node {
  name: "global_step/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "global_step"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "shape"
    value {
      shape {
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "global_step"
    }
  }
}
node {
  name: "global_step/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "global_step"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/Assign"
  op: "AssignVariableOp"
  input: "global_step"
  input: "global_step/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "global_step/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "global_step"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "global_step/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "global_step"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/Switch"
  op: "Switch"
  input: "global_step/VarIsInitializedOp"
  input: "global_step/VarIsInitializedOp"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/switch_t"
  op: "Identity"
  input: "global_step/cond/Switch:1"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/switch_f"
  op: "Identity"
  input: "global_step/cond/Switch"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/pred_id"
  op: "Identity"
  input: "global_step/VarIsInitializedOp"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "global_step/cond/Read/ReadVariableOp/Switch:1"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "global_step/cond/Read/ReadVariableOp/Switch"
  op: "Switch"
  input: "global_step"
  input: "global_step/cond/pred_id"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/Identity"
  op: "Identity"
  input: "global_step/cond/Read/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/Switch_1"
  op: "Switch"
  input: "global_step/Initializer/zeros"
  input: "global_step/cond/pred_id"
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/Merge"
  op: "Merge"
  input: "global_step/cond/Switch_1"
  input: "global_step/cond/Identity"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "global_step/add"
  op: "AddV2"
  input: "global_step/cond/Merge"
  input: "global_step/add/y"
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/Const"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1000
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
          dim {
            size: 1000
          }
        }
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-576-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-965-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-403-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-981-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-301-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-426-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-720-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-4-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-861-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-437-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-181-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-784-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-820-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-410-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-136-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-797-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-582-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-843-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-996-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-9-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-648-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-256-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-795-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-813-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-104-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-588-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-268-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-790-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-40-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-766-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-831-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-563-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-459-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-918-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-530-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-281-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-666-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-927-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-679-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-214-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-478-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-543-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-389-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-873-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-566-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-185-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-805-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-0-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-993-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-494-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-690-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-293-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-60-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-227-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-501-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-333-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-81-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-144-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-997-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-440-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-621-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-735-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-407-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-529-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-228-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-798-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-490-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-776-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-550-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-567-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-328-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-146-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-11-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-473-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-807-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-645-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-629-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-302-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-499-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-689-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-294-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-944-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-173-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-800-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-352-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-49-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-240-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-998-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-461-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-577-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-778-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-647-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-825-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-606-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-420-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-373-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-674-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-840-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-149-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-617-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-914-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-447-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-852-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-188-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-570-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-376-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-213-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-752-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-448-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-487-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-119-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-842-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-165-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-235-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-703-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-253-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-308-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-95-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-836-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-110-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-44-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-740-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-360-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-316-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-961-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-38-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-101-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-292-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-554-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-368-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-526-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-239-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-126-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-574-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-404-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-521-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-340-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-157-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-522-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-751-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-990-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-936-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-62-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-334-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-871-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-488-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-166-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-762-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-387-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-814-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-598-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-603-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-32-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-117-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-917-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-910-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-636-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-425-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-713-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-378-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-875-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-429-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-714-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-564-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-210-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-821-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-995-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-697-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-928-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-179-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-133-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-844-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-496-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-571-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-520-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-950-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-208-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-749-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-497-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-847-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-926-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-668-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-851-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-280-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-441-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-57-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-312-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-980-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-135-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-658-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-632-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-304-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-901-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-561-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-408-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-514-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-615-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-620-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-637-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-482-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-324-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-201-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-673-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-794-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-597-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-625-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-756-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-278-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-379-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-258-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-211-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-212-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-919-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-966-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-832-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-500-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-614-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-339-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-432-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-415-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-528-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-218-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-767-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-739-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-642-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-511-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-894-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-970-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-22-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-460-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-458-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-890-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-604-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-992-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-677-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-502-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-759-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-400-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-54-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-809-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-900-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-132-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-80-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-88-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-470-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-883-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-630-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-353-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-799-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-55-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-651-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-152-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-700-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-421-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-701-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-147-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-225-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-112-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-676-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-903-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-351-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-371-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-733-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-755-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-590-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-162-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-734-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-391-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-150-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-465-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-456-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-783-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-300-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-639-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-177-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-297-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-332-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-56-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-161-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-401-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-848-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-310-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-343-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-802-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-610-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-559-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-412-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-859-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-384-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-383-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-224-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-688-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-35-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-121-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-107-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-665-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-87-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-374-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-834-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-748-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-675-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-826-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-587-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-245-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-472-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-525-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-846-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-287-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-899-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-414-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-148-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-787-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-313-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-396-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-314-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-631-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-471-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-829-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-569-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-823-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-618-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-738-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-266-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-315-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-12-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-558-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-20-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-539-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-291-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-37-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-957-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-411-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-106-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-215-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-416-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-462-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-552-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-344-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-184-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-726-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-544-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-134-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-169-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-533-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-428-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-216-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-259-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-261-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-753-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-77-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-347-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-827-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-671-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-892-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-782-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-198-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-306-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-906-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-535-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-430-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-763-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-349-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-288-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-893-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-729-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-167-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-803-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-702-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-504-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-393-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-862-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-876-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-467-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-477-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-626-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-451-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-972-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-578-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-868-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-889-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-601-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-640-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-750-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-397-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-987-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-531-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-433-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-29-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-24-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-435-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-439-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-540-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-8-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-74-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-768-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-781-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-123-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-660-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-942-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-468-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-760-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-592-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-837-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-923-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-736-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-937-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-591-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-481-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-931-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-41-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-503-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-202-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-485-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-707-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-622-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-33-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-183-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-728-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-247-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-549-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-945-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-839-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-382-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-509-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-565-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-710-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-265-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-599-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-128-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-395-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-553-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-754-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-305-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-924-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-922-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-796-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-808-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-31-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-296-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-804-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-6-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-513-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-48-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-195-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-251-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-406-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-968-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-271-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-114-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-322-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-335-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-230-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-817-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-480-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-295-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-286-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-15-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-974-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-830-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-26-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-607-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-388-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-547-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-747-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-761-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-909-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-695-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-238-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-357-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-381-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-652-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-953-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-176-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-725-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-682-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-160-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-955-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-712-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-170-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-746-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-979-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-573-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-68-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-611-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-222-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-2-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-888-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-78-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-58-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-585-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-273-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-638-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-234-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-50-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-643-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-90-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-380-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-72-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-358-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-236-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-655-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-518-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-495-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-717-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-641-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-537-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-66-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-723-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-399-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-354-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-692-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-139-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-715-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-818-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-483-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-605-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-30-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-985-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-988-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-687-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-860-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-872-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-89-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-858-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-70-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-94-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-698-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-450-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-982-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-911-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-963-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-792-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-952-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-436-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-284-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-207-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-299-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-103-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-880-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-189-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-954-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-285-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-45-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-270-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-678-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-602-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-581-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-492-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-913-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-369-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-681-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-200-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-524-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-223-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-187-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-815-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-764-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-627-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-469-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-102-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-466-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-422-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-596-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-372-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-272-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-361-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-118-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-887-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-98-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-949-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-811-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-650-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-336-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-869-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-594-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-28-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-365-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-457-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-824-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-338-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-773-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-46-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-541-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-932-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-237-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-191-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-956-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-366-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-180-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-780-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-624-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-512-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-986-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-863-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-52-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-670-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-438-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-320-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-3-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-850-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-933-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-774-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-742-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-130-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-721-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-23-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-36-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-446-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-835-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-105-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-10-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-84-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-696-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-793-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-856-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-867-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-877-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-765-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-870-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-879-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-517-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-727-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-841-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-86-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-881-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-741-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-267-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-375-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-941-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-42-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-47-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-274-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-345-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-580-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-298-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-943-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-69-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-327-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-704-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-709-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-252-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-959-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-882-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-667-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-282-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-43-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-449-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-219-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-443-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-454-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-174-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-822-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-19-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-806-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-164-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-964-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-878-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-362-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-978-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-129-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-908-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-355-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-628-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-886-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-946-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-423-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-154-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-644-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-346-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-527-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-85-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-586-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-142-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-498-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-323-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-958-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-685-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-838-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-545-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-732-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-854-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-168-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-683-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-672-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-975-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-452-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-83-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-719-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-612-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-418-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-772-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-193-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-127-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-912-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-445-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-405-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-693-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-246-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-572-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-505-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-556-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-475-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-991-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-196-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-34-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-51-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-5-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-708-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-100-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-262-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-649-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-93-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-39-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-307-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-711-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-16-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-109-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-929-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-257-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-321-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-14-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-691-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-724-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-155-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-560-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-680-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-855-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-515-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-331-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-402-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-145-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-178-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-731-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-59-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-1-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-833-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-659-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-434-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-623-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-718-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-318-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-419-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-646-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-171-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-250-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-231-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-476-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-743-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-896-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-489-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-661-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-204-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-916-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-551-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-279-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-508-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-232-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-63-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-716-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-864-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-97-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-206-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-124-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-493-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-244-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-484-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-455-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-172-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-884-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-76-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-330-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-73-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-608-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-424-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-151-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-593-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-392-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-386-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-967-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-444-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-260-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-589-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-915-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-192-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-350-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-934-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-137-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-849-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-516-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-510-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-115-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-217-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-532-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-657-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-377-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-116-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-583-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-791-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-921-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-199-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-779-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-263-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-722-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-479-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-197-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-546-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-254-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-113-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-664-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-409-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-653-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-771-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-367-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-898-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-994-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-874-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-568-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-866-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-108-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-730-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-534-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-654-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-902-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-656-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-788-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-243-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-812-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-616-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-364-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-816-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-209-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-71-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-99-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-337-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-241-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-895-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-977-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-983-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-584-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-486-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-186-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-233-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-229-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-609-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-777-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-91-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-464-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-290-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-319-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-67-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-507-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-175-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-317-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-801-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-633-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-935-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-523-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-491-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-579-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-264-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-289-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-125-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-13-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-948-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-141-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-442-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-905-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-506-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-453-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-131-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-276-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-140-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-562-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-634-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-619-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-907-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-65-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-595-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-359-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-122-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-938-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-398-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-662-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-120-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-226-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-394-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-775-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-269-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-769-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-940-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-548-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-431-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-575-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-785-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-390-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-329-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-920-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-27-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-64-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-538-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-962-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-971-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-699-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-542-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-969-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-853-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-348-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-891-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-79-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-7-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-363-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-989-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-163-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-356-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-745-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-275-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-474-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-828-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-613-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-999-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-309-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-413-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-947-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-283-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-25-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-686-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-857-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-885-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-557-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-385-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-427-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-810-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-984-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-326-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-519-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-897-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-341-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-156-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-705-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-92-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-182-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-194-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-143-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-75-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-758-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-96-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-600-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-684-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-904-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-694-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-303-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-819-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-220-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-536-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-744-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-635-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-21-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-930-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-82-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-221-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-865-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-311-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-158-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-789-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-277-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-53-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-370-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-159-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-61-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-737-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-973-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-786-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-138-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-153-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-960-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-939-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-325-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-925-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-555-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-757-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-342-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-111-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-845-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-255-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-203-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-951-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-417-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-976-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-18-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-463-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-248-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-17-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-242-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-770-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-669-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-706-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-190-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-205-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-663-of-1000"
        string_val: "/researchdisk/training_dataset_sentences/train_tokenized_512/pretrain_data.tfrecord-249-of-1000"
      }
    }
  }
}
node {
  name: "input_pipeline_task0/TensorSliceDataset"
  op: "TensorSliceDataset"
  input: "input_pipeline_task0/Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "Toutput_types"
    value {
      list {
        type: DT_STRING
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/count"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: -1
      }
    }
  }
}
node {
  name: "input_pipeline_task0/RepeatDataset"
  op: "RepeatDataset"
  input: "input_pipeline_task0/TensorSliceDataset"
  input: "input_pipeline_task0/count"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_STRING
      }
    }
  }
}
node {
  name: "input_pipeline_task0/buffer_size"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 1000
      }
    }
  }
}
node {
  name: "input_pipeline_task0/seed"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "input_pipeline_task0/seed2"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "input_pipeline_task0/ShuffleDataset"
  op: "ShuffleDataset"
  input: "input_pipeline_task0/RepeatDataset"
  input: "input_pipeline_task0/buffer_size"
  input: "input_pipeline_task0/seed"
  input: "input_pipeline_task0/seed2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_STRING
      }
    }
  }
  attr {
    key: "reshuffle_each_iteration"
    value {
      b: true
    }
  }
}
node {
  name: "input_pipeline_task0/cycle_length"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 4
      }
    }
  }
}
node {
  name: "input_pipeline_task0/block_length"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 1
      }
    }
  }
}
node {
  name: "input_pipeline_task0/buffer_output_elements"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 2
      }
    }
  }
}
node {
  name: "input_pipeline_task0/prefetch_input_elements"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 8
      }
    }
  }
}
node {
  name: "input_pipeline_task0/LegacyParallelInterleaveDatasetV2"
  op: "LegacyParallelInterleaveDatasetV2"
  input: "input_pipeline_task0/ShuffleDataset"
  input: "input_pipeline_task0/cycle_length"
  input: "input_pipeline_task0/block_length"
  input: "input_pipeline_task0/buffer_output_elements"
  input: "input_pipeline_task0/prefetch_input_elements"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "Targuments"
    value {
      list {
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "deterministic"
    value {
      s: "false"
    }
  }
  attr {
    key: "f"
    value {
      func {
        name: "__inference_tf_data_experimental_parallel_interleave_classabc.ABCMeta_44"
        attr {
          key: "_tf_data_function"
          value {
            b: true
          }
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_STRING
      }
    }
  }
}
node {
  name: "input_pipeline_task0/buffer_size_1"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 100
      }
    }
  }
}
node {
  name: "input_pipeline_task0/seed_1"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "input_pipeline_task0/seed2_1"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "input_pipeline_task0/ShuffleDataset_1"
  op: "ShuffleDataset"
  input: "input_pipeline_task0/LegacyParallelInterleaveDatasetV2"
  input: "input_pipeline_task0/buffer_size_1"
  input: "input_pipeline_task0/seed_1"
  input: "input_pipeline_task0/seed2_1"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_STRING
      }
    }
  }
  attr {
    key: "reshuffle_each_iteration"
    value {
      b: true
    }
  }
}
node {
  name: "input_pipeline_task0/batch_size"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 32
      }
    }
  }
}
node {
  name: "input_pipeline_task0/num_parallel_calls"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 128
      }
    }
  }
}
node {
  name: "input_pipeline_task0/drop_remainder"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_BOOL
        tensor_shape {
        }
        bool_val: true
      }
    }
  }
}
node {
  name: "input_pipeline_task0/MapAndBatchDataset"
  op: "MapAndBatchDataset"
  input: "input_pipeline_task0/ShuffleDataset_1"
  input: "input_pipeline_task0/batch_size"
  input: "input_pipeline_task0/num_parallel_calls"
  input: "input_pipeline_task0/drop_remainder"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "Targuments"
    value {
      list {
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "f"
    value {
      func {
        name: "__inference_tf_data_experimental_map_and_batch_lambda_72"
        attr {
          key: "_tf_data_function"
          value {
            b: true
          }
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "preserve_cardinality"
    value {
      b: true
    }
  }
}
node {
  name: "input_pipeline_task0/IteratorV2"
  op: "IteratorV2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "input_pipeline_task0/MakeIterator"
  op: "MakeIterator"
  input: "input_pipeline_task0/MapAndBatchDataset"
  input: "input_pipeline_task0/IteratorV2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
}
node {
  name: "input_pipeline_task0/IteratorToStringHandle"
  op: "IteratorToStringHandle"
  input: "input_pipeline_task0/IteratorV2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "tpu_estimator/iterations_per_loop/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "tpu_estimator/iterations_per_loop"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "shape"
    value {
      shape {
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "tpu_estimator/iterations_per_loop"
    }
  }
}
node {
  name: "tpu_estimator/iterations_per_loop/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "tpu_estimator/iterations_per_loop"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "tpu_estimator/iterations_per_loop/Assign"
  op: "AssignVariableOp"
  input: "tpu_estimator/iterations_per_loop"
  input: "tpu_estimator/iterations_per_loop/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "tpu_estimator/iterations_per_loop/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "tpu_estimator/iterations_per_loop"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "input_pipeline_task0/Identity/ReadVariableOp"
  op: "ReadVariableOp"
  input: "tpu_estimator/iterations_per_loop"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "input_pipeline_task0/Identity"
  op: "Identity"
  input: "input_pipeline_task0/Identity/ReadVariableOp"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/Const_1"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/Enter"
  op: "Enter"
  input: "input_pipeline_task0/Const_1"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "input_pipeline_task0/while/while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: false
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "input_pipeline_task0/while/Merge"
  op: "Merge"
  input: "input_pipeline_task0/while/Enter"
  input: "input_pipeline_task0/while/NextIteration"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/Less"
  op: "Less"
  input: "input_pipeline_task0/while/Merge"
  input: "input_pipeline_task0/while/Less/Enter"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/Less/Enter"
  op: "Enter"
  input: "input_pipeline_task0/Identity"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "input_pipeline_task0/while/while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "input_pipeline_task0/while/LoopCond"
  op: "LoopCond"
  input: "input_pipeline_task0/while/Less"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/Switch"
  op: "Switch"
  input: "input_pipeline_task0/while/Merge"
  input: "input_pipeline_task0/while/LoopCond"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/while/Merge"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/Identity"
  op: "Identity"
  input: "input_pipeline_task0/while/Switch:1"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/IteratorGetNext"
  op: "IteratorGetNext"
  input: "input_pipeline_task0/while/IteratorGetNext/Enter"
  input: "^input_pipeline_task0/while/Identity"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/IteratorGetNext/Enter"
  op: "Enter"
  input: "input_pipeline_task0/IteratorV2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "input_pipeline_task0/while/while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "input_pipeline_task0/while/IteratorGetNext_1"
  op: "IteratorGetNext"
  input: "input_pipeline_task0/while/IteratorGetNext/Enter"
  input: "^input_pipeline_task0/while/IteratorGetNext"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/IteratorGetNext_2"
  op: "IteratorGetNext"
  input: "input_pipeline_task0/while/IteratorGetNext/Enter"
  input: "^input_pipeline_task0/while/IteratorGetNext"
  input: "^input_pipeline_task0/while/IteratorGetNext_1"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/IteratorGetNext_3"
  op: "IteratorGetNext"
  input: "input_pipeline_task0/while/IteratorGetNext/Enter"
  input: "^input_pipeline_task0/while/IteratorGetNext"
  input: "^input_pipeline_task0/while/IteratorGetNext_1"
  input: "^input_pipeline_task0/while/IteratorGetNext_2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/IteratorGetNext_4"
  op: "IteratorGetNext"
  input: "input_pipeline_task0/while/IteratorGetNext/Enter"
  input: "^input_pipeline_task0/while/IteratorGetNext"
  input: "^input_pipeline_task0/while/IteratorGetNext_1"
  input: "^input_pipeline_task0/while/IteratorGetNext_2"
  input: "^input_pipeline_task0/while/IteratorGetNext_3"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/IteratorGetNext_5"
  op: "IteratorGetNext"
  input: "input_pipeline_task0/while/IteratorGetNext/Enter"
  input: "^input_pipeline_task0/while/IteratorGetNext"
  input: "^input_pipeline_task0/while/IteratorGetNext_1"
  input: "^input_pipeline_task0/while/IteratorGetNext_2"
  input: "^input_pipeline_task0/while/IteratorGetNext_3"
  input: "^input_pipeline_task0/while/IteratorGetNext_4"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/IteratorGetNext_6"
  op: "IteratorGetNext"
  input: "input_pipeline_task0/while/IteratorGetNext/Enter"
  input: "^input_pipeline_task0/while/IteratorGetNext"
  input: "^input_pipeline_task0/while/IteratorGetNext_1"
  input: "^input_pipeline_task0/while/IteratorGetNext_2"
  input: "^input_pipeline_task0/while/IteratorGetNext_3"
  input: "^input_pipeline_task0/while/IteratorGetNext_4"
  input: "^input_pipeline_task0/while/IteratorGetNext_5"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/IteratorGetNext_7"
  op: "IteratorGetNext"
  input: "input_pipeline_task0/while/IteratorGetNext/Enter"
  input: "^input_pipeline_task0/while/IteratorGetNext"
  input: "^input_pipeline_task0/while/IteratorGetNext_1"
  input: "^input_pipeline_task0/while/IteratorGetNext_2"
  input: "^input_pipeline_task0/while/IteratorGetNext_3"
  input: "^input_pipeline_task0/while/IteratorGetNext_4"
  input: "^input_pipeline_task0/while/IteratorGetNext_5"
  input: "^input_pipeline_task0/while/IteratorGetNext_6"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/InfeedQueue/enqueue/0"
  op: "InfeedEnqueueTuple"
  input: "input_pipeline_task0/while/IteratorGetNext"
  input: "input_pipeline_task0/while/IteratorGetNext:1"
  input: "input_pipeline_task0/while/IteratorGetNext:2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 0
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "layouts"
    value {
      list {
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/InfeedQueue/enqueue/1"
  op: "InfeedEnqueueTuple"
  input: "input_pipeline_task0/while/IteratorGetNext_1"
  input: "input_pipeline_task0/while/IteratorGetNext_1:1"
  input: "input_pipeline_task0/while/IteratorGetNext_1:2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 1
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "layouts"
    value {
      list {
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/InfeedQueue/enqueue/2"
  op: "InfeedEnqueueTuple"
  input: "input_pipeline_task0/while/IteratorGetNext_2"
  input: "input_pipeline_task0/while/IteratorGetNext_2:1"
  input: "input_pipeline_task0/while/IteratorGetNext_2:2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 2
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "layouts"
    value {
      list {
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/InfeedQueue/enqueue/3"
  op: "InfeedEnqueueTuple"
  input: "input_pipeline_task0/while/IteratorGetNext_3"
  input: "input_pipeline_task0/while/IteratorGetNext_3:1"
  input: "input_pipeline_task0/while/IteratorGetNext_3:2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 3
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "layouts"
    value {
      list {
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/InfeedQueue/enqueue/4"
  op: "InfeedEnqueueTuple"
  input: "input_pipeline_task0/while/IteratorGetNext_4"
  input: "input_pipeline_task0/while/IteratorGetNext_4:1"
  input: "input_pipeline_task0/while/IteratorGetNext_4:2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 4
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "layouts"
    value {
      list {
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/InfeedQueue/enqueue/5"
  op: "InfeedEnqueueTuple"
  input: "input_pipeline_task0/while/IteratorGetNext_5"
  input: "input_pipeline_task0/while/IteratorGetNext_5:1"
  input: "input_pipeline_task0/while/IteratorGetNext_5:2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 5
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "layouts"
    value {
      list {
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/InfeedQueue/enqueue/6"
  op: "InfeedEnqueueTuple"
  input: "input_pipeline_task0/while/IteratorGetNext_6"
  input: "input_pipeline_task0/while/IteratorGetNext_6:1"
  input: "input_pipeline_task0/while/IteratorGetNext_6:2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 6
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "layouts"
    value {
      list {
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/InfeedQueue/enqueue/7"
  op: "InfeedEnqueueTuple"
  input: "input_pipeline_task0/while/IteratorGetNext_7"
  input: "input_pipeline_task0/while/IteratorGetNext_7:1"
  input: "input_pipeline_task0/while/IteratorGetNext_7:2"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@input_pipeline_task0/MapAndBatchDataset"
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 7
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "layouts"
    value {
      list {
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/add/y"
  op: "Const"
  input: "^input_pipeline_task0/while/InfeedQueue/enqueue/0"
  input: "^input_pipeline_task0/while/InfeedQueue/enqueue/1"
  input: "^input_pipeline_task0/while/InfeedQueue/enqueue/2"
  input: "^input_pipeline_task0/while/InfeedQueue/enqueue/3"
  input: "^input_pipeline_task0/while/InfeedQueue/enqueue/4"
  input: "^input_pipeline_task0/while/InfeedQueue/enqueue/5"
  input: "^input_pipeline_task0/while/InfeedQueue/enqueue/6"
  input: "^input_pipeline_task0/while/InfeedQueue/enqueue/7"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/add"
  op: "AddV2"
  input: "input_pipeline_task0/while/Identity"
  input: "input_pipeline_task0/while/add/y"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/NextIteration"
  op: "NextIteration"
  input: "input_pipeline_task0/while/add"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "input_pipeline_task0/while/Exit"
  op: "Exit"
  input: "input_pipeline_task0/while/Switch"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "Const"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "Const_1"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Const_2"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "Const_3"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 3
      }
    }
  }
}
node {
  name: "Const_4"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 4
      }
    }
  }
}
node {
  name: "Const_5"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 5
      }
    }
  }
}
node {
  name: "Const_6"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 6
      }
    }
  }
}
node {
  name: "Const_7"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 7
      }
    }
  }
}
node {
  name: "packed"
  op: "Pack"
  input: "Const"
  input: "Const_1"
  input: "Const_2"
  input: "Const_3"
  input: "Const_4"
  input: "Const_5"
  input: "Const_6"
  input: "Const_7"
  attr {
    key: "N"
    value {
      i: 8
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
}
node {
  name: "split/split_dim"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "split"
  op: "Split"
  input: "split/split_dim"
  input: "packed"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
        shape {
          dim {
            size: 1
          }
        }
        shape {
          dim {
            size: 1
          }
        }
        shape {
          dim {
            size: 1
          }
        }
        shape {
          dim {
            size: 1
          }
        }
        shape {
          dim {
            size: 1
          }
        }
        shape {
          dim {
            size: 1
          }
        }
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "num_split"
    value {
      i: 8
    }
  }
}
node {
  name: "input0"
  op: "TPUReplicatedInput"
  input: "split"
  input: "split:1"
  input: "split:2"
  input: "split:3"
  input: "split:4"
  input: "split:5"
  input: "split:6"
  input: "split:7"
  attr {
    key: "N"
    value {
      i: 8
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "index"
    value {
      i: 0
    }
  }
  attr {
    key: "is_mirrored_variable"
    value {
      b: false
    }
  }
  attr {
    key: "is_packed"
    value {
      b: false
    }
  }
}
node {
  name: "cluster/pivot"
  op: "NoOp"
  attr {
    key: "_pivot_for_cluster"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "TPUReplicateMetadata"
  op: "TPUReplicateMetadata"
  input: "^cluster/pivot"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "allow_soft_placement"
    value {
      b: false
    }
  }
  attr {
    key: "computation_shape"
    value {
      list {
      }
    }
  }
  attr {
    key: "device_assignment"
    value {
      list {
      }
    }
  }
  attr {
    key: "host_compute_core"
    value {
      list {
      }
    }
  }
  attr {
    key: "num_cores_per_replica"
    value {
      i: 1
    }
  }
  attr {
    key: "num_replicas"
    value {
      i: 8
    }
  }
  attr {
    key: "padding_map"
    value {
      list {
      }
    }
  }
  attr {
    key: "step_marker_location"
    value {
      s: "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP"
    }
  }
  attr {
    key: "topology"
    value {
      s: ""
    }
  }
  attr {
    key: "use_spmd_for_xla_partitioning"
    value {
      b: true
    }
  }
  attr {
    key: "use_tpu"
    value {
      b: true
    }
  }
}
node {
  name: "replicated_input_0"
  op: "Identity"
  input: "input0"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_input_identity"
    value {
      b: true
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape/shape"
  op: "Const"
  input: "^TPUReplicateMetadata"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "Reshape/shape_1"
  op: "Const"
  input: "^TPUReplicateMetadata"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "Reshape"
  op: "Reshape"
  input: "replicated_input_0"
  input: "Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Const_8"
  op: "Const"
  input: "^TPUReplicateMetadata"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "Const_9"
  op: "Const"
  input: "^TPUReplicateMetadata"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 10000000.0
      }
    }
  }
}
node {
  name: "Enter"
  op: "Enter"
  input: "Const_8"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: false
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Enter_1"
  op: "Enter"
  input: "Const_9"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: false
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Merge"
  op: "Merge"
  input: "Enter"
  input: "NextIteration"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Merge_1"
  op: "Merge"
  input: "Enter_1"
  input: "NextIteration_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Less/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Less/ReadVariableOp/Enter"
  input: "^Merge"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "Less/ReadVariableOp/Enter"
  op: "Enter"
  input: "tpu_estimator/iterations_per_loop"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Less"
  op: "Less"
  input: "Merge"
  input: "Less/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "LoopCond"
  op: "LoopCond"
  input: "Less"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Switch"
  op: "Switch"
  input: "Merge"
  input: "LoopCond"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@Merge"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Switch_1"
  op: "Switch"
  input: "Merge_1"
  input: "LoopCond"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@Merge_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Identity"
  op: "Identity"
  input: "Switch:1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Identity_1"
  op: "Identity"
  input: "Switch_1:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "add"
  op: "AddV2"
  input: "Identity"
  input: "add/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "InfeedQueue/dequeue"
  op: "InfeedDequeueTuple"
  input: "^Identity"
  device: "/device:TPU_REPLICATED_CORE:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
}
node {
  name: "ones_like/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "ones_like/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_BOOL
        tensor_shape {
        }
        bool_val: true
      }
    }
  }
}
node {
  name: "ones_like"
  op: "Fill"
  input: "ones_like/Shape"
  input: "ones_like/Const"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "NotEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 3
      }
    }
  }
}
node {
  name: "NotEqual"
  op: "NotEqual"
  input: "InfeedQueue/dequeue"
  input: "NotEqual/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "incompatible_shape_error"
    value {
      b: true
    }
  }
}
node {
  name: "and"
  op: "LogicalAnd"
  input: "ones_like"
  input: "NotEqual"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "NotEqual_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "NotEqual_1"
  op: "NotEqual"
  input: "InfeedQueue/dequeue"
  input: "NotEqual_1/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "incompatible_shape_error"
    value {
      b: true
    }
  }
}
node {
  name: "and_1"
  op: "LogicalAnd"
  input: "and"
  input: "NotEqual_1"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "NotEqual_2/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 4
      }
    }
  }
}
node {
  name: "NotEqual_2"
  op: "NotEqual"
  input: "InfeedQueue/dequeue"
  input: "NotEqual_2/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "incompatible_shape_error"
    value {
      b: true
    }
  }
}
node {
  name: "and_2"
  op: "LogicalAnd"
  input: "and_1"
  input: "NotEqual_2"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Cast"
  op: "Cast"
  input: "InfeedQueue/dequeue:1"
  attr {
    key: "DstT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "and_3"
  op: "LogicalAnd"
  input: "and_2"
  input: "Cast"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "Sum"
  op: "Sum"
  input: "InfeedQueue/dequeue:1"
  input: "Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "Cast_1"
  op: "Cast"
  input: "Sum"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.15000000596046448
      }
    }
  }
}
node {
  name: "mul"
  op: "Mul"
  input: "Cast_1"
  input: "mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Round"
  op: "Round"
  input: "mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Cast_2"
  op: "Cast"
  input: "Round"
  attr {
    key: "DstT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Minimum/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 79
      }
    }
  }
}
node {
  name: "Minimum"
  op: "Minimum"
  input: "Minimum/x"
  input: "Cast_2"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Maximum/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Maximum"
  op: "Maximum"
  input: "Maximum/x"
  input: "Minimum"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "SequenceMask/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 79
      }
    }
  }
}
node {
  name: "SequenceMask/Const_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "SequenceMask/Const_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "SequenceMask/Range"
  op: "Range"
  input: "SequenceMask/Const_1"
  input: "SequenceMask/Const"
  input: "SequenceMask/Const_2"
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "SequenceMask/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "SequenceMask/ExpandDims"
  op: "ExpandDims"
  input: "Maximum"
  input: "SequenceMask/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "SequenceMask/Cast"
  op: "Cast"
  input: "SequenceMask/ExpandDims"
  attr {
    key: "DstT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "SequenceMask/Less"
  op: "Less"
  input: "SequenceMask/Range"
  input: "SequenceMask/Cast"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Cast_3"
  op: "Cast"
  input: "SequenceMask/Less"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Cast_4"
  op: "Cast"
  input: "and_3"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "mul_1"
  op: "Mul"
  input: "mul_1/x"
  input: "Cast_4"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "Sum_1"
  op: "Sum"
  input: "mul_1"
  input: "Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "truediv"
  op: "RealDiv"
  input: "mul_1"
  input: "Sum_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "StopGradient"
  op: "StopGradient"
  input: "truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Log"
  op: "Log"
  input: "StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "categorical/Multinomial/num_samples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 79
      }
    }
  }
}
node {
  name: "categorical/Multinomial"
  op: "Multinomial"
  input: "Log"
  input: "categorical/Multinomial/num_samples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "output_dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "Cast_5"
  op: "Cast"
  input: "Cast_3"
  attr {
    key: "DstT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2"
  op: "Mul"
  input: "categorical/Multinomial"
  input: "Cast_5"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "range/start"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "range/limit"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 32
      }
    }
  }
}
node {
  name: "range/delta"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "range"
  op: "Range"
  input: "range/start"
  input: "range/limit"
  input: "range/delta"
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_3/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 512
      }
    }
  }
}
node {
  name: "mul_3"
  op: "Mul"
  input: "mul_3/x"
  input: "range"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ExpandDims"
  op: "ExpandDims"
  input: "mul_3"
  input: "ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1"
  op: "AddV2"
  input: "mul_2"
  input: "ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\377\377\377\377\001\000\000\000"
      }
    }
  }
}
node {
  name: "Reshape_1"
  op: "Reshape"
  input: "add_1"
  input: "Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "Reshape_2"
  op: "Reshape"
  input: "InfeedQueue/dequeue"
  input: "Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "GatherNd"
  op: "GatherNd"
  input: "Reshape_2"
  input: "Reshape_1"
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tparams"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\377\377\377\377"
      }
    }
  }
}
node {
  name: "Reshape_3"
  op: "Reshape"
  input: "GatherNd"
  input: "Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Cast_6"
  op: "Cast"
  input: "Cast_3"
  attr {
    key: "DstT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_4"
  op: "Mul"
  input: "Reshape_3"
  input: "Cast_6"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "random_uniform/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000O\000\000\000"
      }
    }
  }
}
node {
  name: "random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "random_uniform/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "Less_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8500000238418579
      }
    }
  }
}
node {
  name: "Less_1"
  op: "Less"
  input: "random_uniform/RandomUniform"
  input: "Less_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Cast_7"
  op: "Cast"
  input: "Less_1"
  attr {
    key: "DstT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_5"
  op: "Mul"
  input: "mul_2"
  input: "Cast_7"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Fill/dims"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000O\000\000\000"
      }
    }
  }
}
node {
  name: "Fill/value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 4
      }
    }
  }
}
node {
  name: "Fill"
  op: "Fill"
  input: "Fill/dims"
  input: "Fill/value"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "ExpandDims_1/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ExpandDims_1"
  op: "ExpandDims"
  input: "InfeedQueue/dequeue"
  input: "ExpandDims_1/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "range_1/start"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "range_1/limit"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 32
      }
    }
  }
}
node {
  name: "range_1/delta"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "range_1"
  op: "Range"
  input: "range_1/start"
  input: "range_1/limit"
  input: "range_1/delta"
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_6/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 512
      }
    }
  }
}
node {
  name: "mul_6"
  op: "Mul"
  input: "mul_6/x"
  input: "range_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ExpandDims_2/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ExpandDims_2"
  op: "ExpandDims"
  input: "mul_6"
  input: "ExpandDims_2/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_2"
  op: "AddV2"
  input: "mul_5"
  input: "ExpandDims_2"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_4/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\377\377\377\377\001\000\000\000"
      }
    }
  }
}
node {
  name: "Reshape_4"
  op: "Reshape"
  input: "add_2"
  input: "Reshape_4/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_5/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\377\377\377\377\001\000\000\000"
      }
    }
  }
}
node {
  name: "Reshape_5"
  op: "Reshape"
  input: "Fill"
  input: "Reshape_5/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ScatterNd/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "ScatterNd"
  op: "ScatterNd"
  input: "Reshape_4"
  input: "Reshape_5"
  input: "ScatterNd/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_6/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "Reshape_6"
  op: "Reshape"
  input: "ScatterNd"
  input: "Reshape_6/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ones/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2528
      }
    }
  }
}
node {
  name: "ones/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "ones"
  op: "Fill"
  input: "ones/shape_as_tensor"
  input: "ones/Const"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "ScatterNd_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 16384
      }
    }
  }
}
node {
  name: "ScatterNd_1"
  op: "ScatterNd"
  input: "Reshape_4"
  input: "ones"
  input: "ScatterNd_1/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_7/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "Reshape_7"
  op: "Reshape"
  input: "ScatterNd_1"
  input: "Reshape_7/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "zeros"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "ones_1/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\377\001\000\000"
      }
    }
  }
}
node {
  name: "ones_1/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "ones_1"
  op: "Fill"
  input: "ones_1/shape_as_tensor"
  input: "ones_1/Const"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 511
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "concat/axis"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "concat"
  op: "ConcatV2"
  input: "zeros"
  input: "ones_1"
  input: "concat/axis"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_7"
  op: "Mul"
  input: "Reshape_7"
  input: "concat"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ExpandDims_3/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ExpandDims_3"
  op: "ExpandDims"
  input: "mul_7"
  input: "ExpandDims_3/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Maximum_1/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Maximum_1"
  op: "Maximum"
  input: "Maximum_1/x"
  input: "ExpandDims_3"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "floordiv"
  op: "FloorDiv"
  input: "Reshape_6"
  input: "Maximum_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Minimum_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Minimum_1"
  op: "Minimum"
  input: "mul_7"
  input: "Minimum_1/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Minimum_2/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Minimum_2"
  op: "Minimum"
  input: "ExpandDims_3"
  input: "Minimum_2/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "sub"
  op: "Sub"
  input: "sub/x"
  input: "Minimum_2"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_8"
  op: "Mul"
  input: "sub"
  input: "ExpandDims_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_9"
  op: "Mul"
  input: "Minimum_2"
  input: "floordiv"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_3"
  op: "AddV2"
  input: "mul_8"
  input: "mul_9"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Squeeze"
  op: "Squeeze"
  input: "add_3"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "squeeze_dims"
    value {
      list {
        i: -1
      }
    }
  }
}
node {
  name: "StopGradient_1"
  op: "StopGradient"
  input: "Squeeze"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "electra/embeddings/ExpandDims"
  op: "ExpandDims"
  input: "StopGradient_1"
  input: "electra/embeddings/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "Y\304\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/embeddings/word_embeddings/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/embeddings/word_embeddings/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/embeddings/word_embeddings/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/embeddings/word_embeddings/Initializer/truncated_normal/mul"
  input: "electra/embeddings/word_embeddings/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 50265
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings/word_embeddings"
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/word_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings/word_embeddings"
  input: "electra/embeddings/word_embeddings/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/word_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "electra/embeddings/Reshape"
  op: "Reshape"
  input: "electra/embeddings/ExpandDims"
  input: "electra/embeddings/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings/one_hot/on_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/embeddings/one_hot/off_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings/one_hot/depth"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 50265
      }
    }
  }
}
node {
  name: "electra/embeddings/one_hot"
  op: "OneHot"
  input: "electra/embeddings/Reshape"
  input: "electra/embeddings/one_hot/depth"
  input: "electra/embeddings/one_hot/on_value"
  input: "electra/embeddings/one_hot/off_value"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "TI"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: -1
    }
  }
}
node {
  name: "electra/embeddings/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings/word_embeddings"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/embeddings/MatMul"
  op: "MatMul"
  input: "electra/embeddings/one_hot"
  input: "electra/embeddings/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/embeddings/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings/Reshape_1"
  op: "Reshape"
  input: "electra/embeddings/MatMul"
  input: "electra/embeddings/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\002\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal/mul"
  input: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 2
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings/token_type_embeddings"
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/token_type_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings/token_type_embeddings"
  input: "electra/embeddings/token_type_embeddings/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/token_type_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "electra/embeddings_1/Reshape"
  op: "Reshape"
  input: "InfeedQueue/dequeue:2"
  input: "electra/embeddings_1/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/one_hot/on_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/embeddings_1/one_hot/off_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_1/one_hot/depth"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "electra/embeddings_1/one_hot"
  op: "OneHot"
  input: "electra/embeddings_1/Reshape"
  input: "electra/embeddings_1/one_hot/depth"
  input: "electra/embeddings_1/one_hot/on_value"
  input: "electra/embeddings_1/one_hot/off_value"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "TI"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: -1
    }
  }
}
node {
  name: "electra/embeddings_1/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings/token_type_embeddings"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/embeddings_1/MatMul"
  op: "MatMul"
  input: "electra/embeddings_1/one_hot"
  input: "electra/embeddings_1/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/embeddings_1/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings_1/Reshape_1"
  op: "Reshape"
  input: "electra/embeddings_1/MatMul"
  input: "electra/embeddings_1/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/add"
  op: "AddV2"
  input: "electra/embeddings/Reshape_1"
  input: "electra/embeddings_1/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 512
      }
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 512
      }
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/LessEqual"
  op: "LessEqual"
  input: "electra/embeddings_1/assert_less_equal/x"
  input: "electra/embeddings_1/assert_less_equal/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/Rank"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/range/start"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/range/delta"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/range"
  op: "Range"
  input: "electra/embeddings_1/assert_less_equal/range/start"
  input: "electra/embeddings_1/assert_less_equal/Rank"
  input: "electra/embeddings_1/assert_less_equal/range/delta"
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/All"
  op: "All"
  input: "electra/embeddings_1/assert_less_equal/LessEqual"
  input: "electra/embeddings_1/assert_less_equal/range"
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/Assert/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "Condition x <= y did not hold element-wise:"
      }
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/Assert/Const_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "x (electra/embeddings_1/assert_less_equal/x:0) = "
      }
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/Assert/Const_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "y (electra/embeddings_1/assert_less_equal/y:0) = "
      }
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/Assert/Assert/data_0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "Condition x <= y did not hold element-wise:"
      }
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/Assert/Assert/data_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "x (electra/embeddings_1/assert_less_equal/x:0) = "
      }
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/Assert/Assert/data_3"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "y (electra/embeddings_1/assert_less_equal/y:0) = "
      }
    }
  }
}
node {
  name: "electra/embeddings_1/assert_less_equal/Assert/Assert"
  op: "Assert"
  input: "electra/embeddings_1/assert_less_equal/All"
  input: "electra/embeddings_1/assert_less_equal/Assert/Assert/data_0"
  input: "electra/embeddings_1/assert_less_equal/Assert/Assert/data_1"
  input: "electra/embeddings_1/assert_less_equal/x"
  input: "electra/embeddings_1/assert_less_equal/Assert/Assert/data_3"
  input: "electra/embeddings_1/assert_less_equal/y"
  attr {
    key: "T"
    value {
      list {
        type: DT_STRING
        type: DT_STRING
        type: DT_INT32
        type: DT_STRING
        type: DT_INT32
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "summarize"
    value {
      i: 3
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/embeddings/position_embeddings/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/embeddings/position_embeddings/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/embeddings/position_embeddings/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/embeddings/position_embeddings/Initializer/truncated_normal/mul"
  input: "electra/embeddings/position_embeddings/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 512
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings/position_embeddings"
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/position_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings/position_embeddings"
  input: "electra/embeddings/position_embeddings/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/position_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/Slice/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/Slice/ReadVariableOp/Enter"
  input: "^electra/embeddings_1/assert_less_equal/Assert/Assert"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/Slice/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings/position_embeddings"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/embeddings_1/Slice/begin"
  op: "Const"
  input: "^electra/embeddings_1/assert_less_equal/Assert/Assert"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\000\000\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings_1/Slice/size"
  op: "Const"
  input: "^electra/embeddings_1/assert_less_equal/Assert/Assert"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\377\377\377\377"
      }
    }
  }
}
node {
  name: "electra/embeddings_1/Slice"
  op: "Slice"
  input: "electra/embeddings_1/Slice/ReadVariableOp"
  input: "electra/embeddings_1/Slice/begin"
  input: "electra/embeddings_1/Slice/size"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/Reshape_2/shape"
  op: "Const"
  input: "^electra/embeddings_1/assert_less_equal/Assert/Assert"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings_1/Reshape_2"
  op: "Reshape"
  input: "electra/embeddings_1/Slice"
  input: "electra/embeddings_1/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/add_1"
  op: "AddV2"
  input: "electra/embeddings_1/add"
  input: "electra/embeddings_1/Reshape_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_1/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma"
  input: "electra/embeddings_1/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_1/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta"
  input: "electra/embeddings_1/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/embeddings_1/add_1"
  input: "electra/embeddings_1/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/embeddings_1/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/embeddings_1/add_1"
  input: "electra/embeddings_1/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/embeddings_1/LayerNorm/moments/SquaredDifference"
  input: "electra/embeddings_1/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/embeddings_1/LayerNorm/moments/variance"
  input: "electra/embeddings_1/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/embeddings_1/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_1/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/embeddings_1/LayerNorm/batchnorm/Rsqrt"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/embeddings_1/add_1"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/embeddings_1/LayerNorm/moments/mean"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_1/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/embeddings_1/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul_1"
  input: "electra/embeddings_1/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/embeddings_1/dropout/Mul"
  op: "Mul"
  input: "electra/embeddings_1/LayerNorm/batchnorm/add_1"
  input: "electra/embeddings_1/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings_1/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/embeddings_1/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/embeddings_1/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/embeddings_1/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/embeddings_1/dropout/random_uniform/RandomUniform"
  input: "electra/embeddings_1/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/dropout/Cast"
  op: "Cast"
  input: "electra/embeddings_1/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_1/dropout/Mul_1"
  op: "Mul"
  input: "electra/embeddings_1/dropout/Mul"
  input: "electra/embeddings_1/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/Initializer/random_uniform/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/Initializer/random_uniform/min"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -0.0765465572476387
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/Initializer/random_uniform/max"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0765465572476387
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/Initializer/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/embeddings_project/kernel/Initializer/random_uniform/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/Initializer/random_uniform/sub"
  op: "Sub"
  input: "generator/embeddings_project/kernel/Initializer/random_uniform/max"
  input: "generator/embeddings_project/kernel/Initializer/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/Initializer/random_uniform/mul"
  op: "Mul"
  input: "generator/embeddings_project/kernel/Initializer/random_uniform/RandomUniform"
  input: "generator/embeddings_project/kernel/Initializer/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/Initializer/random_uniform"
  op: "AddV2"
  input: "generator/embeddings_project/kernel/Initializer/random_uniform/mul"
  input: "generator/embeddings_project/kernel/Initializer/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/embeddings_project/kernel"
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/kernel"
  input: "generator/embeddings_project/kernel/Initializer/random_uniform"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/embeddings_project/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/embeddings_project/bias"
    }
  }
}
node {
  name: "generator/embeddings_project/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/embeddings_project/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/bias"
  input: "generator/embeddings_project/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/Tensordot/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/Tensordot/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/embeddings_project/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/embeddings_project/Tensordot/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "generator/embeddings_project/Tensordot/Reshape"
  op: "Reshape"
  input: "electra/embeddings_1/dropout/Mul_1"
  input: "generator/embeddings_project/Tensordot/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/embeddings_project/Tensordot/MatMul"
  op: "MatMul"
  input: "generator/embeddings_project/Tensordot/Reshape"
  input: "generator/embeddings_project/Tensordot/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/embeddings_project/Tensordot/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/embeddings_project/Tensordot"
  op: "Reshape"
  input: "generator/embeddings_project/Tensordot/MatMul"
  input: "generator/embeddings_project/Tensordot/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/embeddings_project/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/embeddings_project/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/embeddings_project/BiasAdd"
  op: "BiasAdd"
  input: "generator/embeddings_project/Tensordot"
  input: "generator/embeddings_project/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape"
  op: "Reshape"
  input: "InfeedQueue/dequeue:1"
  input: "generator/encoder/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Cast"
  op: "Cast"
  input: "generator/encoder/Reshape"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/ones/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/ones/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/ones"
  op: "Fill"
  input: "generator/encoder/ones/shape_as_tensor"
  input: "generator/encoder/ones/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/mul"
  op: "Mul"
  input: "generator/encoder/ones"
  input: "generator/encoder/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\377\377\377\377\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_1"
  op: "Reshape"
  input: "generator/embeddings_project/BiasAdd"
  input: "generator/encoder/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel"
  input: "generator/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias"
  input: "generator/encoder/layer_0/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/Reshape_1"
  input: "generator/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_0/attention/self/query/MatMul"
  input: "generator/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel"
  input: "generator/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias"
  input: "generator/encoder/layer_0/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/Reshape_1"
  input: "generator/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_0/attention/self/key/MatMul"
  input: "generator/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel"
  input: "generator/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias"
  input: "generator/encoder/layer_0/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/Reshape_1"
  input: "generator/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_0/attention/self/value/MatMul"
  input: "generator/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_0/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_0/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_0/attention/self/Reshape"
  input: "generator/encoder/layer_0/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_0/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_0/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_0/attention/self/Reshape_1"
  input: "generator/encoder/layer_0/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_0/attention/self/transpose"
  input: "generator/encoder/layer_0/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/self/MatMul"
  input: "generator/encoder/layer_0/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_0/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_0/attention/self/sub/x"
  input: "generator/encoder/layer_0/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/self/sub"
  input: "generator/encoder/layer_0/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_0/attention/self/Mul"
  input: "generator/encoder/layer_0/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_0/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/self/Softmax"
  input: "generator/encoder/layer_0/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_0/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_0/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_0/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_0/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/self/dropout/Mul"
  input: "generator/encoder/layer_0/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_0/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_0/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_0/attention/self/Reshape_2"
  input: "generator/encoder/layer_0/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_0/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_0/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_0/attention/self/MatMul_1"
  input: "generator/encoder/layer_0/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_0/attention/self/transpose_3"
  input: "generator/encoder/layer_0/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias"
  input: "generator/encoder/layer_0/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_0/attention/self/Reshape_3"
  input: "generator/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_0/attention/output/dense/MatMul"
  input: "generator/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_0/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_0/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_0/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_0/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_0/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/output/dropout/Mul"
  input: "generator/encoder/layer_0/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_0/attention/output/dropout/Mul_1"
  input: "generator/encoder/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_0/attention/output/add"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_0/attention/output/add"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/output/add"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias"
  input: "generator/encoder/layer_0/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_0/intermediate/dense/MatMul"
  input: "generator/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_0/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_0/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_0/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_0/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_0/intermediate/dense/add/x"
  input: "generator/encoder/layer_0/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_0/intermediate/dense/mul/x"
  input: "generator/encoder/layer_0/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_0/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel"
  input: "generator/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias"
  input: "generator/encoder/layer_0/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_0/intermediate/dense/mul_1"
  input: "generator/encoder/layer_0/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_0/output/dense/MatMul"
  input: "generator/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_0/output/dense/BiasAdd"
  input: "generator/encoder/layer_0/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_0/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_0/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_0/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_0/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/output/dropout/Mul"
  input: "generator/encoder/layer_0/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_0/output/dropout/Mul_1"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_0/output/add"
  input: "generator/encoder/layer_0/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_0/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_0/output/add"
  input: "generator/encoder/layer_0/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_0/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_0/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/output/add"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_0/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel"
  input: "generator/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias"
  input: "generator/encoder/layer_1/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_1/attention/self/query/MatMul"
  input: "generator/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel"
  input: "generator/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias"
  input: "generator/encoder/layer_1/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_1/attention/self/key/MatMul"
  input: "generator/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel"
  input: "generator/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias"
  input: "generator/encoder/layer_1/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_1/attention/self/value/MatMul"
  input: "generator/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_1/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_1/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_1/attention/self/Reshape"
  input: "generator/encoder/layer_1/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_1/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_1/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_1/attention/self/Reshape_1"
  input: "generator/encoder/layer_1/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_1/attention/self/transpose"
  input: "generator/encoder/layer_1/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/self/MatMul"
  input: "generator/encoder/layer_1/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_1/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_1/attention/self/sub/x"
  input: "generator/encoder/layer_1/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/self/sub"
  input: "generator/encoder/layer_1/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_1/attention/self/Mul"
  input: "generator/encoder/layer_1/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_1/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/self/Softmax"
  input: "generator/encoder/layer_1/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_1/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_1/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_1/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_1/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/self/dropout/Mul"
  input: "generator/encoder/layer_1/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_1/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_1/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_1/attention/self/Reshape_2"
  input: "generator/encoder/layer_1/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_1/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_1/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_1/attention/self/MatMul_1"
  input: "generator/encoder/layer_1/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_1/attention/self/transpose_3"
  input: "generator/encoder/layer_1/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias"
  input: "generator/encoder/layer_1/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_1/attention/self/Reshape_3"
  input: "generator/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_1/attention/output/dense/MatMul"
  input: "generator/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_1/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_1/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_1/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_1/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_1/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/output/dropout/Mul"
  input: "generator/encoder/layer_1/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_1/attention/output/dropout/Mul_1"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_1/attention/output/add"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_1/attention/output/add"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/output/add"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias"
  input: "generator/encoder/layer_1/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_1/intermediate/dense/MatMul"
  input: "generator/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_1/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_1/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_1/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_1/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_1/intermediate/dense/add/x"
  input: "generator/encoder/layer_1/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_1/intermediate/dense/mul/x"
  input: "generator/encoder/layer_1/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_1/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel"
  input: "generator/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias"
  input: "generator/encoder/layer_1/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_1/intermediate/dense/mul_1"
  input: "generator/encoder/layer_1/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_1/output/dense/MatMul"
  input: "generator/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_1/output/dense/BiasAdd"
  input: "generator/encoder/layer_1/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_1/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_1/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_1/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_1/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/output/dropout/Mul"
  input: "generator/encoder/layer_1/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_1/output/dropout/Mul_1"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_1/output/add"
  input: "generator/encoder/layer_1/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_1/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_1/output/add"
  input: "generator/encoder/layer_1/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_1/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_1/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/output/add"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_1/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel"
  input: "generator/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias"
  input: "generator/encoder/layer_2/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_2/attention/self/query/MatMul"
  input: "generator/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel"
  input: "generator/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias"
  input: "generator/encoder/layer_2/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_2/attention/self/key/MatMul"
  input: "generator/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel"
  input: "generator/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias"
  input: "generator/encoder/layer_2/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_2/attention/self/value/MatMul"
  input: "generator/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_2/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_2/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_2/attention/self/Reshape"
  input: "generator/encoder/layer_2/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_2/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_2/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_2/attention/self/Reshape_1"
  input: "generator/encoder/layer_2/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_2/attention/self/transpose"
  input: "generator/encoder/layer_2/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/self/MatMul"
  input: "generator/encoder/layer_2/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_2/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_2/attention/self/sub/x"
  input: "generator/encoder/layer_2/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/self/sub"
  input: "generator/encoder/layer_2/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_2/attention/self/Mul"
  input: "generator/encoder/layer_2/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_2/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/self/Softmax"
  input: "generator/encoder/layer_2/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_2/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_2/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_2/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_2/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/self/dropout/Mul"
  input: "generator/encoder/layer_2/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_2/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_2/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_2/attention/self/Reshape_2"
  input: "generator/encoder/layer_2/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_2/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_2/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_2/attention/self/MatMul_1"
  input: "generator/encoder/layer_2/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_2/attention/self/transpose_3"
  input: "generator/encoder/layer_2/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias"
  input: "generator/encoder/layer_2/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_2/attention/self/Reshape_3"
  input: "generator/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_2/attention/output/dense/MatMul"
  input: "generator/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_2/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_2/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_2/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_2/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_2/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/output/dropout/Mul"
  input: "generator/encoder/layer_2/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_2/attention/output/dropout/Mul_1"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_2/attention/output/add"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_2/attention/output/add"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/output/add"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias"
  input: "generator/encoder/layer_2/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_2/intermediate/dense/MatMul"
  input: "generator/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_2/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_2/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_2/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_2/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_2/intermediate/dense/add/x"
  input: "generator/encoder/layer_2/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_2/intermediate/dense/mul/x"
  input: "generator/encoder/layer_2/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_2/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel"
  input: "generator/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias"
  input: "generator/encoder/layer_2/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_2/intermediate/dense/mul_1"
  input: "generator/encoder/layer_2/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_2/output/dense/MatMul"
  input: "generator/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_2/output/dense/BiasAdd"
  input: "generator/encoder/layer_2/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_2/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_2/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_2/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_2/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/output/dropout/Mul"
  input: "generator/encoder/layer_2/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_2/output/dropout/Mul_1"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_2/output/add"
  input: "generator/encoder/layer_2/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_2/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_2/output/add"
  input: "generator/encoder/layer_2/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_2/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_2/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/output/add"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_2/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel"
  input: "generator/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias"
  input: "generator/encoder/layer_3/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_3/attention/self/query/MatMul"
  input: "generator/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel"
  input: "generator/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias"
  input: "generator/encoder/layer_3/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_3/attention/self/key/MatMul"
  input: "generator/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel"
  input: "generator/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias"
  input: "generator/encoder/layer_3/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_3/attention/self/value/MatMul"
  input: "generator/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_3/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_3/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_3/attention/self/Reshape"
  input: "generator/encoder/layer_3/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_3/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_3/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_3/attention/self/Reshape_1"
  input: "generator/encoder/layer_3/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_3/attention/self/transpose"
  input: "generator/encoder/layer_3/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/self/MatMul"
  input: "generator/encoder/layer_3/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_3/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_3/attention/self/sub/x"
  input: "generator/encoder/layer_3/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/self/sub"
  input: "generator/encoder/layer_3/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_3/attention/self/Mul"
  input: "generator/encoder/layer_3/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_3/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/self/Softmax"
  input: "generator/encoder/layer_3/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_3/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_3/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_3/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_3/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/self/dropout/Mul"
  input: "generator/encoder/layer_3/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_3/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_3/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_3/attention/self/Reshape_2"
  input: "generator/encoder/layer_3/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_3/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_3/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_3/attention/self/MatMul_1"
  input: "generator/encoder/layer_3/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_3/attention/self/transpose_3"
  input: "generator/encoder/layer_3/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias"
  input: "generator/encoder/layer_3/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_3/attention/self/Reshape_3"
  input: "generator/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_3/attention/output/dense/MatMul"
  input: "generator/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_3/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_3/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_3/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_3/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_3/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/output/dropout/Mul"
  input: "generator/encoder/layer_3/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_3/attention/output/dropout/Mul_1"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_3/attention/output/add"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_3/attention/output/add"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/output/add"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias"
  input: "generator/encoder/layer_3/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_3/intermediate/dense/MatMul"
  input: "generator/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_3/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_3/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_3/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_3/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_3/intermediate/dense/add/x"
  input: "generator/encoder/layer_3/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_3/intermediate/dense/mul/x"
  input: "generator/encoder/layer_3/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_3/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel"
  input: "generator/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias"
  input: "generator/encoder/layer_3/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_3/intermediate/dense/mul_1"
  input: "generator/encoder/layer_3/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_3/output/dense/MatMul"
  input: "generator/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_3/output/dense/BiasAdd"
  input: "generator/encoder/layer_3/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_3/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_3/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_3/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_3/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/output/dropout/Mul"
  input: "generator/encoder/layer_3/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_3/output/dropout/Mul_1"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_3/output/add"
  input: "generator/encoder/layer_3/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_3/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_3/output/add"
  input: "generator/encoder/layer_3/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_3/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_3/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/output/add"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_3/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel"
  input: "generator/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias"
  input: "generator/encoder/layer_4/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_4/attention/self/query/MatMul"
  input: "generator/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel"
  input: "generator/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias"
  input: "generator/encoder/layer_4/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_4/attention/self/key/MatMul"
  input: "generator/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel"
  input: "generator/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias"
  input: "generator/encoder/layer_4/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_4/attention/self/value/MatMul"
  input: "generator/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_4/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_4/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_4/attention/self/Reshape"
  input: "generator/encoder/layer_4/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_4/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_4/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_4/attention/self/Reshape_1"
  input: "generator/encoder/layer_4/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_4/attention/self/transpose"
  input: "generator/encoder/layer_4/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/self/MatMul"
  input: "generator/encoder/layer_4/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_4/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_4/attention/self/sub/x"
  input: "generator/encoder/layer_4/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/self/sub"
  input: "generator/encoder/layer_4/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_4/attention/self/Mul"
  input: "generator/encoder/layer_4/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_4/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/self/Softmax"
  input: "generator/encoder/layer_4/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_4/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_4/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_4/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_4/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/self/dropout/Mul"
  input: "generator/encoder/layer_4/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_4/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_4/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_4/attention/self/Reshape_2"
  input: "generator/encoder/layer_4/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_4/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_4/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_4/attention/self/MatMul_1"
  input: "generator/encoder/layer_4/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_4/attention/self/transpose_3"
  input: "generator/encoder/layer_4/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias"
  input: "generator/encoder/layer_4/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_4/attention/self/Reshape_3"
  input: "generator/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_4/attention/output/dense/MatMul"
  input: "generator/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_4/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_4/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_4/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_4/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_4/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/output/dropout/Mul"
  input: "generator/encoder/layer_4/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_4/attention/output/dropout/Mul_1"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_4/attention/output/add"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_4/attention/output/add"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/output/add"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias"
  input: "generator/encoder/layer_4/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_4/intermediate/dense/MatMul"
  input: "generator/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_4/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_4/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_4/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_4/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_4/intermediate/dense/add/x"
  input: "generator/encoder/layer_4/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_4/intermediate/dense/mul/x"
  input: "generator/encoder/layer_4/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_4/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel"
  input: "generator/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias"
  input: "generator/encoder/layer_4/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_4/intermediate/dense/mul_1"
  input: "generator/encoder/layer_4/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_4/output/dense/MatMul"
  input: "generator/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_4/output/dense/BiasAdd"
  input: "generator/encoder/layer_4/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_4/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_4/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_4/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_4/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/output/dropout/Mul"
  input: "generator/encoder/layer_4/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_4/output/dropout/Mul_1"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_4/output/add"
  input: "generator/encoder/layer_4/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_4/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_4/output/add"
  input: "generator/encoder/layer_4/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_4/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_4/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/output/add"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_4/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel"
  input: "generator/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias"
  input: "generator/encoder/layer_5/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_5/attention/self/query/MatMul"
  input: "generator/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel"
  input: "generator/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias"
  input: "generator/encoder/layer_5/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_5/attention/self/key/MatMul"
  input: "generator/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel"
  input: "generator/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias"
  input: "generator/encoder/layer_5/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_5/attention/self/value/MatMul"
  input: "generator/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_5/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_5/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_5/attention/self/Reshape"
  input: "generator/encoder/layer_5/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_5/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_5/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_5/attention/self/Reshape_1"
  input: "generator/encoder/layer_5/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_5/attention/self/transpose"
  input: "generator/encoder/layer_5/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/self/MatMul"
  input: "generator/encoder/layer_5/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_5/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_5/attention/self/sub/x"
  input: "generator/encoder/layer_5/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/self/sub"
  input: "generator/encoder/layer_5/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_5/attention/self/Mul"
  input: "generator/encoder/layer_5/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_5/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/self/Softmax"
  input: "generator/encoder/layer_5/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_5/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_5/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_5/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_5/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/self/dropout/Mul"
  input: "generator/encoder/layer_5/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_5/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_5/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_5/attention/self/Reshape_2"
  input: "generator/encoder/layer_5/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_5/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_5/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_5/attention/self/MatMul_1"
  input: "generator/encoder/layer_5/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_5/attention/self/transpose_3"
  input: "generator/encoder/layer_5/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias"
  input: "generator/encoder/layer_5/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_5/attention/self/Reshape_3"
  input: "generator/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_5/attention/output/dense/MatMul"
  input: "generator/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_5/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_5/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_5/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_5/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_5/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/output/dropout/Mul"
  input: "generator/encoder/layer_5/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_5/attention/output/dropout/Mul_1"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_5/attention/output/add"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_5/attention/output/add"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/output/add"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias"
  input: "generator/encoder/layer_5/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_5/intermediate/dense/MatMul"
  input: "generator/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_5/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_5/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_5/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_5/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_5/intermediate/dense/add/x"
  input: "generator/encoder/layer_5/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_5/intermediate/dense/mul/x"
  input: "generator/encoder/layer_5/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_5/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel"
  input: "generator/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias"
  input: "generator/encoder/layer_5/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_5/intermediate/dense/mul_1"
  input: "generator/encoder/layer_5/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_5/output/dense/MatMul"
  input: "generator/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_5/output/dense/BiasAdd"
  input: "generator/encoder/layer_5/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_5/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_5/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_5/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_5/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/output/dropout/Mul"
  input: "generator/encoder/layer_5/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_5/output/dropout/Mul_1"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_5/output/add"
  input: "generator/encoder/layer_5/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_5/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_5/output/add"
  input: "generator/encoder/layer_5/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_5/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_5/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/output/add"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_5/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel"
  input: "generator/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias"
  input: "generator/encoder/layer_6/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_6/attention/self/query/MatMul"
  input: "generator/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel"
  input: "generator/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias"
  input: "generator/encoder/layer_6/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_6/attention/self/key/MatMul"
  input: "generator/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel"
  input: "generator/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias"
  input: "generator/encoder/layer_6/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_6/attention/self/value/MatMul"
  input: "generator/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_6/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_6/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_6/attention/self/Reshape"
  input: "generator/encoder/layer_6/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_6/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_6/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_6/attention/self/Reshape_1"
  input: "generator/encoder/layer_6/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_6/attention/self/transpose"
  input: "generator/encoder/layer_6/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/self/MatMul"
  input: "generator/encoder/layer_6/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_6/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_6/attention/self/sub/x"
  input: "generator/encoder/layer_6/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/self/sub"
  input: "generator/encoder/layer_6/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_6/attention/self/Mul"
  input: "generator/encoder/layer_6/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_6/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/self/Softmax"
  input: "generator/encoder/layer_6/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_6/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_6/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_6/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_6/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/self/dropout/Mul"
  input: "generator/encoder/layer_6/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_6/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_6/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_6/attention/self/Reshape_2"
  input: "generator/encoder/layer_6/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_6/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_6/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_6/attention/self/MatMul_1"
  input: "generator/encoder/layer_6/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_6/attention/self/transpose_3"
  input: "generator/encoder/layer_6/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias"
  input: "generator/encoder/layer_6/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_6/attention/self/Reshape_3"
  input: "generator/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_6/attention/output/dense/MatMul"
  input: "generator/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_6/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_6/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_6/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_6/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_6/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/output/dropout/Mul"
  input: "generator/encoder/layer_6/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_6/attention/output/dropout/Mul_1"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_6/attention/output/add"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_6/attention/output/add"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/output/add"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias"
  input: "generator/encoder/layer_6/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_6/intermediate/dense/MatMul"
  input: "generator/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_6/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_6/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_6/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_6/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_6/intermediate/dense/add/x"
  input: "generator/encoder/layer_6/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_6/intermediate/dense/mul/x"
  input: "generator/encoder/layer_6/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_6/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel"
  input: "generator/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias"
  input: "generator/encoder/layer_6/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_6/intermediate/dense/mul_1"
  input: "generator/encoder/layer_6/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_6/output/dense/MatMul"
  input: "generator/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_6/output/dense/BiasAdd"
  input: "generator/encoder/layer_6/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_6/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_6/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_6/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_6/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/output/dropout/Mul"
  input: "generator/encoder/layer_6/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_6/output/dropout/Mul_1"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_6/output/add"
  input: "generator/encoder/layer_6/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_6/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_6/output/add"
  input: "generator/encoder/layer_6/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_6/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_6/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/output/add"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_6/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel"
  input: "generator/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias"
  input: "generator/encoder/layer_7/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_7/attention/self/query/MatMul"
  input: "generator/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel"
  input: "generator/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias"
  input: "generator/encoder/layer_7/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_7/attention/self/key/MatMul"
  input: "generator/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel"
  input: "generator/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias"
  input: "generator/encoder/layer_7/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_7/attention/self/value/MatMul"
  input: "generator/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_7/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_7/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_7/attention/self/Reshape"
  input: "generator/encoder/layer_7/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_7/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_7/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_7/attention/self/Reshape_1"
  input: "generator/encoder/layer_7/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_7/attention/self/transpose"
  input: "generator/encoder/layer_7/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/self/MatMul"
  input: "generator/encoder/layer_7/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_7/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_7/attention/self/sub/x"
  input: "generator/encoder/layer_7/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/self/sub"
  input: "generator/encoder/layer_7/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_7/attention/self/Mul"
  input: "generator/encoder/layer_7/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_7/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/self/Softmax"
  input: "generator/encoder/layer_7/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_7/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_7/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_7/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_7/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/self/dropout/Mul"
  input: "generator/encoder/layer_7/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_7/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_7/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_7/attention/self/Reshape_2"
  input: "generator/encoder/layer_7/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_7/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_7/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_7/attention/self/MatMul_1"
  input: "generator/encoder/layer_7/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_7/attention/self/transpose_3"
  input: "generator/encoder/layer_7/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias"
  input: "generator/encoder/layer_7/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_7/attention/self/Reshape_3"
  input: "generator/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_7/attention/output/dense/MatMul"
  input: "generator/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_7/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_7/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_7/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_7/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_7/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/output/dropout/Mul"
  input: "generator/encoder/layer_7/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_7/attention/output/dropout/Mul_1"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_7/attention/output/add"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_7/attention/output/add"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/output/add"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias"
  input: "generator/encoder/layer_7/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_7/intermediate/dense/MatMul"
  input: "generator/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_7/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_7/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_7/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_7/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_7/intermediate/dense/add/x"
  input: "generator/encoder/layer_7/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_7/intermediate/dense/mul/x"
  input: "generator/encoder/layer_7/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_7/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel"
  input: "generator/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias"
  input: "generator/encoder/layer_7/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_7/intermediate/dense/mul_1"
  input: "generator/encoder/layer_7/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_7/output/dense/MatMul"
  input: "generator/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_7/output/dense/BiasAdd"
  input: "generator/encoder/layer_7/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_7/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_7/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_7/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_7/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/output/dropout/Mul"
  input: "generator/encoder/layer_7/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_7/output/dropout/Mul_1"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_7/output/add"
  input: "generator/encoder/layer_7/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_7/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_7/output/add"
  input: "generator/encoder/layer_7/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_7/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_7/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/output/add"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_7/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel"
  input: "generator/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias"
  input: "generator/encoder/layer_8/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_8/attention/self/query/MatMul"
  input: "generator/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel"
  input: "generator/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias"
  input: "generator/encoder/layer_8/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_8/attention/self/key/MatMul"
  input: "generator/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel"
  input: "generator/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias"
  input: "generator/encoder/layer_8/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_8/attention/self/value/MatMul"
  input: "generator/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_8/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_8/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_8/attention/self/Reshape"
  input: "generator/encoder/layer_8/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_8/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_8/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_8/attention/self/Reshape_1"
  input: "generator/encoder/layer_8/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_8/attention/self/transpose"
  input: "generator/encoder/layer_8/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/self/MatMul"
  input: "generator/encoder/layer_8/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_8/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_8/attention/self/sub/x"
  input: "generator/encoder/layer_8/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/self/sub"
  input: "generator/encoder/layer_8/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_8/attention/self/Mul"
  input: "generator/encoder/layer_8/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_8/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/self/Softmax"
  input: "generator/encoder/layer_8/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_8/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_8/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_8/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_8/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/self/dropout/Mul"
  input: "generator/encoder/layer_8/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_8/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_8/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_8/attention/self/Reshape_2"
  input: "generator/encoder/layer_8/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_8/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_8/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_8/attention/self/MatMul_1"
  input: "generator/encoder/layer_8/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_8/attention/self/transpose_3"
  input: "generator/encoder/layer_8/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias"
  input: "generator/encoder/layer_8/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_8/attention/self/Reshape_3"
  input: "generator/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_8/attention/output/dense/MatMul"
  input: "generator/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_8/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_8/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_8/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_8/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_8/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/output/dropout/Mul"
  input: "generator/encoder/layer_8/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_8/attention/output/dropout/Mul_1"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_8/attention/output/add"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_8/attention/output/add"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/output/add"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias"
  input: "generator/encoder/layer_8/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_8/intermediate/dense/MatMul"
  input: "generator/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_8/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_8/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_8/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_8/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_8/intermediate/dense/add/x"
  input: "generator/encoder/layer_8/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_8/intermediate/dense/mul/x"
  input: "generator/encoder/layer_8/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_8/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel"
  input: "generator/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias"
  input: "generator/encoder/layer_8/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_8/intermediate/dense/mul_1"
  input: "generator/encoder/layer_8/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_8/output/dense/MatMul"
  input: "generator/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_8/output/dense/BiasAdd"
  input: "generator/encoder/layer_8/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_8/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_8/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_8/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_8/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/output/dropout/Mul"
  input: "generator/encoder/layer_8/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_8/output/dropout/Mul_1"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_8/output/add"
  input: "generator/encoder/layer_8/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_8/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_8/output/add"
  input: "generator/encoder/layer_8/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_8/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_8/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/output/add"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_8/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel"
  input: "generator/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias"
  input: "generator/encoder/layer_9/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_9/attention/self/query/MatMul"
  input: "generator/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel"
  input: "generator/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias"
  input: "generator/encoder/layer_9/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_9/attention/self/key/MatMul"
  input: "generator/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel"
  input: "generator/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias"
  input: "generator/encoder/layer_9/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_9/attention/self/value/MatMul"
  input: "generator/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_9/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_9/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_9/attention/self/Reshape"
  input: "generator/encoder/layer_9/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_9/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_9/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_9/attention/self/Reshape_1"
  input: "generator/encoder/layer_9/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_9/attention/self/transpose"
  input: "generator/encoder/layer_9/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/self/MatMul"
  input: "generator/encoder/layer_9/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_9/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_9/attention/self/sub/x"
  input: "generator/encoder/layer_9/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/self/sub"
  input: "generator/encoder/layer_9/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_9/attention/self/Mul"
  input: "generator/encoder/layer_9/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_9/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/self/Softmax"
  input: "generator/encoder/layer_9/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_9/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_9/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_9/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_9/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/self/dropout/Mul"
  input: "generator/encoder/layer_9/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_9/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_9/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_9/attention/self/Reshape_2"
  input: "generator/encoder/layer_9/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_9/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_9/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_9/attention/self/MatMul_1"
  input: "generator/encoder/layer_9/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_9/attention/self/transpose_3"
  input: "generator/encoder/layer_9/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias"
  input: "generator/encoder/layer_9/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_9/attention/self/Reshape_3"
  input: "generator/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_9/attention/output/dense/MatMul"
  input: "generator/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_9/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_9/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_9/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_9/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_9/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/output/dropout/Mul"
  input: "generator/encoder/layer_9/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_9/attention/output/dropout/Mul_1"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_9/attention/output/add"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_9/attention/output/add"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/output/add"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias"
  input: "generator/encoder/layer_9/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_9/intermediate/dense/MatMul"
  input: "generator/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_9/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_9/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_9/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_9/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_9/intermediate/dense/add/x"
  input: "generator/encoder/layer_9/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_9/intermediate/dense/mul/x"
  input: "generator/encoder/layer_9/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_9/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel"
  input: "generator/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias"
  input: "generator/encoder/layer_9/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_9/intermediate/dense/mul_1"
  input: "generator/encoder/layer_9/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_9/output/dense/MatMul"
  input: "generator/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_9/output/dense/BiasAdd"
  input: "generator/encoder/layer_9/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_9/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_9/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_9/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_9/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/output/dropout/Mul"
  input: "generator/encoder/layer_9/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_9/output/dropout/Mul_1"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_9/output/add"
  input: "generator/encoder/layer_9/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_9/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_9/output/add"
  input: "generator/encoder/layer_9/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_9/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_9/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/output/add"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_9/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel"
  input: "generator/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias"
  input: "generator/encoder/layer_10/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_10/attention/self/query/MatMul"
  input: "generator/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel"
  input: "generator/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias"
  input: "generator/encoder/layer_10/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_10/attention/self/key/MatMul"
  input: "generator/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel"
  input: "generator/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias"
  input: "generator/encoder/layer_10/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_10/attention/self/value/MatMul"
  input: "generator/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_10/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_10/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_10/attention/self/Reshape"
  input: "generator/encoder/layer_10/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_10/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_10/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_10/attention/self/Reshape_1"
  input: "generator/encoder/layer_10/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_10/attention/self/transpose"
  input: "generator/encoder/layer_10/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/self/MatMul"
  input: "generator/encoder/layer_10/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_10/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_10/attention/self/sub/x"
  input: "generator/encoder/layer_10/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/self/sub"
  input: "generator/encoder/layer_10/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_10/attention/self/Mul"
  input: "generator/encoder/layer_10/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_10/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/self/Softmax"
  input: "generator/encoder/layer_10/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_10/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_10/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_10/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_10/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/self/dropout/Mul"
  input: "generator/encoder/layer_10/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_10/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_10/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_10/attention/self/Reshape_2"
  input: "generator/encoder/layer_10/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_10/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_10/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_10/attention/self/MatMul_1"
  input: "generator/encoder/layer_10/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_10/attention/self/transpose_3"
  input: "generator/encoder/layer_10/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias"
  input: "generator/encoder/layer_10/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_10/attention/self/Reshape_3"
  input: "generator/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_10/attention/output/dense/MatMul"
  input: "generator/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_10/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_10/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_10/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_10/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_10/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/output/dropout/Mul"
  input: "generator/encoder/layer_10/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_10/attention/output/dropout/Mul_1"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_10/attention/output/add"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_10/attention/output/add"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/output/add"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias"
  input: "generator/encoder/layer_10/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_10/intermediate/dense/MatMul"
  input: "generator/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_10/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_10/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_10/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_10/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_10/intermediate/dense/add/x"
  input: "generator/encoder/layer_10/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_10/intermediate/dense/mul/x"
  input: "generator/encoder/layer_10/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_10/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel"
  input: "generator/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias"
  input: "generator/encoder/layer_10/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_10/intermediate/dense/mul_1"
  input: "generator/encoder/layer_10/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_10/output/dense/MatMul"
  input: "generator/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_10/output/dense/BiasAdd"
  input: "generator/encoder/layer_10/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_10/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_10/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_10/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_10/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/output/dropout/Mul"
  input: "generator/encoder/layer_10/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_10/output/dropout/Mul_1"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_10/output/add"
  input: "generator/encoder/layer_10/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_10/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_10/output/add"
  input: "generator/encoder/layer_10/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_10/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_10/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/output/add"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_10/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/query/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel"
  input: "generator/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/query/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias"
  input: "generator/encoder/layer_11/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_11/attention/self/query/MatMul"
  input: "generator/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/key/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel"
  input: "generator/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/key/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias"
  input: "generator/encoder/layer_11/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_11/attention/self/key/MatMul"
  input: "generator/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/value/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel"
  input: "generator/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/value/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias"
  input: "generator/encoder/layer_11/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_11/attention/self/value/MatMul"
  input: "generator/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/Reshape"
  op: "Reshape"
  input: "generator/encoder/layer_11/attention/self/query/BiasAdd"
  input: "generator/encoder/layer_11/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/transpose"
  op: "Transpose"
  input: "generator/encoder/layer_11/attention/self/Reshape"
  input: "generator/encoder/layer_11/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/layer_11/attention/self/key/BiasAdd"
  input: "generator/encoder/layer_11/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/transpose_1"
  op: "Transpose"
  input: "generator/encoder/layer_11/attention/self/Reshape_1"
  input: "generator/encoder/layer_11/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_11/attention/self/transpose"
  input: "generator/encoder/layer_11/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/Mul"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/self/MatMul"
  input: "generator/encoder/layer_11/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "generator/encoder/mul"
  input: "generator/encoder/layer_11/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/sub"
  op: "Sub"
  input: "generator/encoder/layer_11/attention/self/sub/x"
  input: "generator/encoder/layer_11/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/self/sub"
  input: "generator/encoder/layer_11/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/add"
  op: "AddV2"
  input: "generator/encoder/layer_11/attention/self/Mul"
  input: "generator/encoder/layer_11/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/Softmax"
  op: "Softmax"
  input: "generator/encoder/layer_11/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/self/Softmax"
  input: "generator/encoder/layer_11/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_11/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_11/attention/self/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_11/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_11/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/self/dropout/Mul"
  input: "generator/encoder/layer_11/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_11/attention/self/value/BiasAdd"
  input: "generator/encoder/layer_11/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/transpose_2"
  op: "Transpose"
  input: "generator/encoder/layer_11/attention/self/Reshape_2"
  input: "generator/encoder/layer_11/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_11/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_11/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/transpose_3"
  op: "Transpose"
  input: "generator/encoder/layer_11/attention/self/MatMul_1"
  input: "generator/encoder/layer_11/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_11/attention/self/transpose_3"
  input: "generator/encoder/layer_11/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias"
  input: "generator/encoder/layer_11/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_11/attention/self/Reshape_3"
  input: "generator/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_11/attention/output/dense/MatMul"
  input: "generator/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/output/dense/BiasAdd"
  input: "generator/encoder/layer_11/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_11/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_11/attention/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_11/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_11/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/output/dropout/Mul"
  input: "generator/encoder/layer_11/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_11/attention/output/dropout/Mul_1"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_11/attention/output/add"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_11/attention/output/add"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/output/add"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/intermediate/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/intermediate/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias"
  input: "generator/encoder/layer_11/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_11/intermediate/dense/MatMul"
  input: "generator/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "generator/encoder/layer_11/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/truediv"
  op: "RealDiv"
  input: "generator/encoder/layer_11/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_11/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/Erf"
  op: "Erf"
  input: "generator/encoder/layer_11/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/add"
  op: "AddV2"
  input: "generator/encoder/layer_11/intermediate/dense/add/x"
  input: "generator/encoder/layer_11/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/mul"
  op: "Mul"
  input: "generator/encoder/layer_11/intermediate/dense/mul/x"
  input: "generator/encoder/layer_11/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/intermediate/dense/BiasAdd"
  input: "generator/encoder/layer_11/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/dense/kernel"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel"
  input: "generator/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/dense/bias"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias"
  input: "generator/encoder/layer_11/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/MatMul"
  op: "MatMul"
  input: "generator/encoder/layer_11/intermediate/dense/mul_1"
  input: "generator/encoder/layer_11/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator/encoder/layer_11/output/dense/MatMul"
  input: "generator/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dropout/Mul"
  op: "Mul"
  input: "generator/encoder/layer_11/output/dense/BiasAdd"
  input: "generator/encoder/layer_11/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "generator/encoder/layer_11/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "generator/encoder/layer_11/output/dropout/random_uniform/RandomUniform"
  input: "generator/encoder/layer_11/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dropout/Cast"
  op: "Cast"
  input: "generator/encoder/layer_11/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dropout/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/output/dropout/Mul"
  input: "generator/encoder/layer_11/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/add"
  op: "AddV2"
  input: "generator/encoder/layer_11/output/dropout/Mul_1"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/LayerNorm/beta"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator/encoder/layer_11/output/add"
  input: "generator/encoder/layer_11/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator/encoder/layer_11/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator/encoder/layer_11/output/add"
  input: "generator/encoder/layer_11/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference"
  input: "generator/encoder/layer_11/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator/encoder/layer_11/output/LayerNorm/moments/variance"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/output/add"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator/encoder/layer_11/output/LayerNorm/moments/mean"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_1"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/stack"
  op: "Pack"
  input: "generator/encoder/layer_0/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_1/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_2/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_3/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_4/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_5/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_6/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_7/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_8/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_9/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_10/attention/self/dropout/Mul_1"
  input: "generator/encoder/layer_11/attention/self/dropout/Mul_1"
  attr {
    key: "N"
    value {
      i: 12
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 12
          }
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_2"
  op: "Reshape"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_3"
  op: "Reshape"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_4/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_4"
  op: "Reshape"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_4/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_5/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_5"
  op: "Reshape"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_5/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_6/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_6"
  op: "Reshape"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_6/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_7/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_7"
  op: "Reshape"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_7/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_8/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_8"
  op: "Reshape"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_8/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_9/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_9"
  op: "Reshape"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_9/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_10/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_10"
  op: "Reshape"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_10/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_11/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_11"
  op: "Reshape"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_11/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_12/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_12"
  op: "Reshape"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_12/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/Reshape_13/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/Reshape_13"
  op: "Reshape"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/add_1"
  input: "generator/encoder/Reshape_13/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator/encoder/stack_1"
  op: "Pack"
  input: "generator/encoder/Reshape_2"
  input: "generator/encoder/Reshape_3"
  input: "generator/encoder/Reshape_4"
  input: "generator/encoder/Reshape_5"
  input: "generator/encoder/Reshape_6"
  input: "generator/encoder/Reshape_7"
  input: "generator/encoder/Reshape_8"
  input: "generator/encoder/Reshape_9"
  input: "generator/encoder/Reshape_10"
  input: "generator/encoder/Reshape_11"
  input: "generator/encoder/Reshape_12"
  input: "generator/encoder/Reshape_13"
  attr {
    key: "N"
    value {
      i: 12
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 12
          }
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
}
node {
  name: "generator/encoder/strided_slice/stack"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "generator/encoder/strided_slice/stack_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "generator/encoder/strided_slice/stack_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator/encoder/strided_slice"
  op: "StridedSlice"
  input: "generator/encoder/stack_1"
  input: "generator/encoder/strided_slice/stack"
  input: "generator/encoder/strided_slice/stack_1"
  input: "generator/encoder/strided_slice/stack_2"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 1
    }
  }
}
node {
  name: "generator/encoder/strided_slice_1/stack"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\000\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/strided_slice_1/stack_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/strided_slice_1/stack_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/strided_slice_1"
  op: "StridedSlice"
  input: "generator/encoder/strided_slice"
  input: "generator/encoder/strided_slice_1/stack"
  input: "generator/encoder/strided_slice_1/stack_1"
  input: "generator/encoder/strided_slice_1/stack_2"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 1
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 1
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 2
    }
  }
}
node {
  name: "generator_predictions/range/start"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "generator_predictions/range/limit"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 32
      }
    }
  }
}
node {
  name: "generator_predictions/range/delta"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "generator_predictions/range"
  op: "Range"
  input: "generator_predictions/range/start"
  input: "generator_predictions/range/limit"
  input: "generator_predictions/range/delta"
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 512
      }
    }
  }
}
node {
  name: "generator_predictions/mul"
  op: "Mul"
  input: "generator_predictions/mul/x"
  input: "generator_predictions/range"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "generator_predictions/ExpandDims"
  op: "ExpandDims"
  input: "generator_predictions/mul"
  input: "generator_predictions/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/add"
  op: "AddV2"
  input: "mul_2"
  input: "generator_predictions/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "generator_predictions/Reshape"
  op: "Reshape"
  input: "generator_predictions/add"
  input: "generator_predictions/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator_predictions/Reshape_1"
  op: "Reshape"
  input: "generator/encoder/strided_slice"
  input: "generator_predictions/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/GatherV2/axis"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "generator_predictions/GatherV2"
  op: "GatherV2"
  input: "generator_predictions/Reshape_1"
  input: "generator_predictions/Reshape"
  input: "generator_predictions/GatherV2/axis"
  attr {
    key: "Taxis"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tparams"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "batch_dims"
    value {
      i: 0
    }
  }
}
node {
  name: "generator_predictions/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\377\377\377\377\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator_predictions/Reshape_2"
  op: "Reshape"
  input: "generator_predictions/GatherV2"
  input: "generator_predictions/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "generator_predictions/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "generator_predictions/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "generator_predictions/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "generator_predictions/dense/kernel/Initializer/truncated_normal/mul"
  input: "generator_predictions/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/dense/kernel"
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/kernel"
  input: "generator_predictions/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/dense/bias"
    }
  }
}
node {
  name: "generator_predictions/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/bias"
  input: "generator_predictions/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/Tensordot/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator_predictions/dense/Tensordot/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\340\t\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator_predictions/dense/Tensordot/Reshape"
  op: "Reshape"
  input: "generator_predictions/Reshape_2"
  input: "generator_predictions/dense/Tensordot/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/dense/Tensordot/MatMul"
  op: "MatMul"
  input: "generator_predictions/dense/Tensordot/Reshape"
  input: "generator_predictions/dense/Tensordot/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "generator_predictions/dense/Tensordot/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "generator_predictions/dense/Tensordot"
  op: "Reshape"
  input: "generator_predictions/dense/Tensordot/MatMul"
  input: "generator_predictions/dense/Tensordot/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator_predictions/dense/BiasAdd"
  op: "BiasAdd"
  input: "generator_predictions/dense/Tensordot"
  input: "generator_predictions/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator_predictions/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "generator_predictions/dense/Sqrt"
  op: "Sqrt"
  input: "generator_predictions/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/dense/truediv"
  op: "RealDiv"
  input: "generator_predictions/dense/BiasAdd"
  input: "generator_predictions/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/dense/Erf"
  op: "Erf"
  input: "generator_predictions/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator_predictions/dense/add"
  op: "AddV2"
  input: "generator_predictions/dense/add/x"
  input: "generator_predictions/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "generator_predictions/dense/mul"
  op: "Mul"
  input: "generator_predictions/dense/mul/x"
  input: "generator_predictions/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/dense/mul_1"
  op: "Mul"
  input: "generator_predictions/dense/BiasAdd"
  input: "generator_predictions/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/LayerNorm/gamma"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/gamma"
  input: "generator_predictions/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/LayerNorm/beta"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/beta"
  input: "generator_predictions/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/moments/mean"
  op: "Mean"
  input: "generator_predictions/dense/mul_1"
  input: "generator_predictions/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "generator_predictions/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "generator_predictions/dense/mul_1"
  input: "generator_predictions/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/moments/variance"
  op: "Mean"
  input: "generator_predictions/LayerNorm/moments/SquaredDifference"
  input: "generator_predictions/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "generator_predictions/LayerNorm/moments/variance"
  input: "generator_predictions/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "generator_predictions/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "generator_predictions/LayerNorm/batchnorm/Rsqrt"
  input: "generator_predictions/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "generator_predictions/dense/mul_1"
  input: "generator_predictions/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "generator_predictions/LayerNorm/moments/mean"
  input: "generator_predictions/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "generator_predictions/LayerNorm/batchnorm/ReadVariableOp"
  input: "generator_predictions/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "generator_predictions/LayerNorm/batchnorm/mul_1"
  input: "generator_predictions/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/output_bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 50265
      }
    }
  }
}
node {
  name: "generator_predictions/output_bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/output_bias/Initializer/zeros"
  op: "Fill"
  input: "generator_predictions/output_bias/Initializer/zeros/shape_as_tensor"
  input: "generator_predictions/output_bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator_predictions/output_bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 50265
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/output_bias"
    }
  }
}
node {
  name: "generator_predictions/output_bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/output_bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/output_bias/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/output_bias"
  input: "generator_predictions/output_bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/output_bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/output_bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/MatMul"
  op: "BatchMatMulV2"
  input: "generator_predictions/LayerNorm/batchnorm/add_1"
  input: "generator_predictions/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "generator_predictions/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/output_bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "generator_predictions/BiasAdd"
  op: "BiasAdd"
  input: "generator_predictions/MatMul"
  input: "generator_predictions/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "generator_predictions/one_hot/on_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "generator_predictions/one_hot/off_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/one_hot/depth"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 50265
      }
    }
  }
}
node {
  name: "generator_predictions/one_hot"
  op: "OneHot"
  input: "mul_4"
  input: "generator_predictions/one_hot/depth"
  input: "generator_predictions/one_hot/on_value"
  input: "generator_predictions/one_hot/off_value"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "TI"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: -1
    }
  }
}
node {
  name: "generator_predictions/ArgMax/dimension"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "generator_predictions/ArgMax"
  op: "ArgMax"
  input: "generator_predictions/BiasAdd"
  input: "generator_predictions/ArgMax/dimension"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "output_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator_predictions/Softmax"
  op: "Softmax"
  input: "generator_predictions/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/LogSoftmax"
  op: "LogSoftmax"
  input: "generator_predictions/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/mul_1"
  op: "Mul"
  input: "generator_predictions/LogSoftmax"
  input: "generator_predictions/one_hot"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "generator_predictions/Sum"
  op: "Sum"
  input: "generator_predictions/mul_1"
  input: "generator_predictions/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "generator_predictions/Neg"
  op: "Neg"
  input: "generator_predictions/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/mul_2"
  op: "Mul"
  input: "Cast_3"
  input: "generator_predictions/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "generator_predictions/Sum_1"
  op: "Sum"
  input: "generator_predictions/mul_2"
  input: "generator_predictions/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "generator_predictions/Const_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "generator_predictions/Sum_2"
  op: "Sum"
  input: "Cast_3"
  input: "generator_predictions/Const_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "generator_predictions/add_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "generator_predictions/add_1"
  op: "AddV2"
  input: "generator_predictions/Sum_2"
  input: "generator_predictions/add_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "generator_predictions/truediv"
  op: "RealDiv"
  input: "generator_predictions/Sum_1"
  input: "generator_predictions/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ExpandDims_4/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ExpandDims_4"
  op: "ExpandDims"
  input: "StopGradient_1"
  input: "ExpandDims_4/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "range_2/start"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "range_2/limit"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 32
      }
    }
  }
}
node {
  name: "range_2/delta"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "range_2"
  op: "Range"
  input: "range_2/start"
  input: "range_2/limit"
  input: "range_2/delta"
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_10/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 512
      }
    }
  }
}
node {
  name: "mul_10"
  op: "Mul"
  input: "mul_10/x"
  input: "range_2"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ExpandDims_5/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ExpandDims_5"
  op: "ExpandDims"
  input: "mul_10"
  input: "ExpandDims_5/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_4"
  op: "AddV2"
  input: "mul_2"
  input: "ExpandDims_5"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_8/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\377\377\377\377\001\000\000\000"
      }
    }
  }
}
node {
  name: "Reshape_8"
  op: "Reshape"
  input: "add_4"
  input: "Reshape_8/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_9/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\377\377\377\377\001\000\000\000"
      }
    }
  }
}
node {
  name: "Reshape_9"
  op: "Reshape"
  input: "mul_4"
  input: "Reshape_9/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ScatterNd_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "ScatterNd_2"
  op: "ScatterNd"
  input: "Reshape_8"
  input: "Reshape_9"
  input: "ScatterNd_2/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_10/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "Reshape_10"
  op: "Reshape"
  input: "ScatterNd_2"
  input: "Reshape_10/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ones_2/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2528
      }
    }
  }
}
node {
  name: "ones_2/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "ones_2"
  op: "Fill"
  input: "ones_2/shape_as_tensor"
  input: "ones_2/Const"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "ScatterNd_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 16384
      }
    }
  }
}
node {
  name: "ScatterNd_3"
  op: "ScatterNd"
  input: "Reshape_8"
  input: "ones_2"
  input: "ScatterNd_3/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_11/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "Reshape_11"
  op: "Reshape"
  input: "ScatterNd_3"
  input: "Reshape_11/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "zeros_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "ones_3/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\377\001\000\000"
      }
    }
  }
}
node {
  name: "ones_3/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "ones_3"
  op: "Fill"
  input: "ones_3/shape_as_tensor"
  input: "ones_3/Const"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 511
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "concat_1/axis"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "concat_1"
  op: "ConcatV2"
  input: "zeros_1"
  input: "ones_3"
  input: "concat_1/axis"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_11"
  op: "Mul"
  input: "Reshape_11"
  input: "concat_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ExpandDims_6/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ExpandDims_6"
  op: "ExpandDims"
  input: "mul_11"
  input: "ExpandDims_6/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Maximum_2/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Maximum_2"
  op: "Maximum"
  input: "Maximum_2/x"
  input: "ExpandDims_6"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "floordiv_1"
  op: "FloorDiv"
  input: "Reshape_10"
  input: "Maximum_2"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Minimum_3/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Minimum_3"
  op: "Minimum"
  input: "mul_11"
  input: "Minimum_3/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Minimum_4/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Minimum_4"
  op: "Minimum"
  input: "ExpandDims_6"
  input: "Minimum_4/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "sub_1/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "sub_1"
  op: "Sub"
  input: "sub_1/x"
  input: "Minimum_4"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_12"
  op: "Mul"
  input: "sub_1"
  input: "ExpandDims_4"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_13"
  op: "Mul"
  input: "Minimum_4"
  input: "floordiv_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_5"
  op: "AddV2"
  input: "mul_12"
  input: "mul_13"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Squeeze_1"
  op: "Squeeze"
  input: "add_5"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "squeeze_dims"
    value {
      list {
        i: -1
      }
    }
  }
}
node {
  name: "truediv_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "truediv_1"
  op: "RealDiv"
  input: "generator_predictions/BiasAdd"
  input: "truediv_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "random_uniform_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000Y\304\000\000"
      }
    }
  }
}
node {
  name: "random_uniform_1/RandomUniform"
  op: "RandomUniform"
  input: "random_uniform_1/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "add_6/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999717180685e-10
      }
    }
  }
}
node {
  name: "add_6"
  op: "AddV2"
  input: "random_uniform_1/RandomUniform"
  input: "add_6/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Log_1"
  op: "Log"
  input: "add_6"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Neg"
  op: "Neg"
  input: "Log_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_7/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999717180685e-10
      }
    }
  }
}
node {
  name: "add_7"
  op: "AddV2"
  input: "Neg"
  input: "add_7/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Log_2"
  op: "Log"
  input: "add_7"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Neg_1"
  op: "Neg"
  input: "Log_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_8"
  op: "AddV2"
  input: "truediv_1"
  input: "Neg_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Softmax"
  op: "Softmax"
  input: "add_8"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ArgMax/dimension"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ArgMax"
  op: "ArgMax"
  input: "Softmax"
  input: "ArgMax/dimension"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "output_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "one_hot/on_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "one_hot/off_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "one_hot/depth"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 50265
      }
    }
  }
}
node {
  name: "one_hot"
  op: "OneHot"
  input: "ArgMax"
  input: "one_hot/depth"
  input: "one_hot/on_value"
  input: "one_hot/off_value"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "TI"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: -1
    }
  }
}
node {
  name: "StopGradient_2"
  op: "StopGradient"
  input: "one_hot"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ArgMax_1/dimension"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ArgMax_1"
  op: "ArgMax"
  input: "StopGradient_2"
  input: "ArgMax_1/dimension"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "output_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "ExpandDims_7/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ExpandDims_7"
  op: "ExpandDims"
  input: "Squeeze_1"
  input: "ExpandDims_7/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "range_3/start"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "range_3/limit"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 32
      }
    }
  }
}
node {
  name: "range_3/delta"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "range_3"
  op: "Range"
  input: "range_3/start"
  input: "range_3/limit"
  input: "range_3/delta"
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_14/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 512
      }
    }
  }
}
node {
  name: "mul_14"
  op: "Mul"
  input: "mul_14/x"
  input: "range_3"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ExpandDims_8/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ExpandDims_8"
  op: "ExpandDims"
  input: "mul_14"
  input: "ExpandDims_8/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_9"
  op: "AddV2"
  input: "mul_2"
  input: "ExpandDims_8"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_12/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\377\377\377\377\001\000\000\000"
      }
    }
  }
}
node {
  name: "Reshape_12"
  op: "Reshape"
  input: "add_9"
  input: "Reshape_12/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_13/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\377\377\377\377\001\000\000\000"
      }
    }
  }
}
node {
  name: "Reshape_13"
  op: "Reshape"
  input: "ArgMax_1"
  input: "Reshape_13/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ScatterNd_4/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "ScatterNd_4"
  op: "ScatterNd"
  input: "Reshape_12"
  input: "Reshape_13"
  input: "ScatterNd_4/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_14/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "Reshape_14"
  op: "Reshape"
  input: "ScatterNd_4"
  input: "Reshape_14/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ones_4/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2528
      }
    }
  }
}
node {
  name: "ones_4/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "ones_4"
  op: "Fill"
  input: "ones_4/shape_as_tensor"
  input: "ones_4/Const"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "ScatterNd_5/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 16384
      }
    }
  }
}
node {
  name: "ScatterNd_5"
  op: "ScatterNd"
  input: "Reshape_12"
  input: "ones_4"
  input: "ScatterNd_5/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_15/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "Reshape_15"
  op: "Reshape"
  input: "ScatterNd_5"
  input: "Reshape_15/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "zeros_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "ones_5/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\377\001\000\000"
      }
    }
  }
}
node {
  name: "ones_5/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "ones_5"
  op: "Fill"
  input: "ones_5/shape_as_tensor"
  input: "ones_5/Const"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 511
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "concat_2/axis"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "concat_2"
  op: "ConcatV2"
  input: "zeros_2"
  input: "ones_5"
  input: "concat_2/axis"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_15"
  op: "Mul"
  input: "Reshape_15"
  input: "concat_2"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ExpandDims_9/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ExpandDims_9"
  op: "ExpandDims"
  input: "mul_15"
  input: "ExpandDims_9/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Maximum_3/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Maximum_3"
  op: "Maximum"
  input: "Maximum_3/x"
  input: "ExpandDims_9"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "floordiv_2"
  op: "FloorDiv"
  input: "Reshape_14"
  input: "Maximum_3"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Minimum_5/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Minimum_5"
  op: "Minimum"
  input: "mul_15"
  input: "Minimum_5/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Minimum_6/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Minimum_6"
  op: "Minimum"
  input: "ExpandDims_9"
  input: "Minimum_6/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "sub_2/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "sub_2"
  op: "Sub"
  input: "sub_2/x"
  input: "Minimum_6"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_16"
  op: "Mul"
  input: "sub_2"
  input: "ExpandDims_7"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_17"
  op: "Mul"
  input: "Minimum_6"
  input: "floordiv_2"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_10"
  op: "AddV2"
  input: "mul_16"
  input: "mul_17"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Squeeze_2"
  op: "Squeeze"
  input: "add_10"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "squeeze_dims"
    value {
      list {
        i: -1
      }
    }
  }
}
node {
  name: "Equal"
  op: "Equal"
  input: "Squeeze_2"
  input: "Squeeze_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "incompatible_shape_error"
    value {
      b: true
    }
  }
}
node {
  name: "Cast_8"
  op: "Cast"
  input: "Equal"
  attr {
    key: "DstT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "sub_3/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "sub_3"
  op: "Sub"
  input: "sub_3/x"
  input: "Cast_8"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_18"
  op: "Mul"
  input: "Minimum_5"
  input: "sub_3"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_19/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "mul_19"
  op: "Mul"
  input: "mul_19/x"
  input: "generator_predictions/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_2/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "electra/embeddings_2/ExpandDims"
  op: "ExpandDims"
  input: "Squeeze_2"
  input: "electra/embeddings_2/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_2/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "electra/embeddings_2/Reshape"
  op: "Reshape"
  input: "electra/embeddings_2/ExpandDims"
  input: "electra/embeddings_2/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_2/one_hot/on_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/embeddings_2/one_hot/off_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_2/one_hot/depth"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 50265
      }
    }
  }
}
node {
  name: "electra/embeddings_2/one_hot"
  op: "OneHot"
  input: "electra/embeddings_2/Reshape"
  input: "electra/embeddings_2/one_hot/depth"
  input: "electra/embeddings_2/one_hot/on_value"
  input: "electra/embeddings_2/one_hot/off_value"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "TI"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: -1
    }
  }
}
node {
  name: "electra/embeddings_2/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_2/MatMul"
  op: "MatMul"
  input: "electra/embeddings_2/one_hot"
  input: "electra/embeddings_2/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/embeddings_2/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings_2/Reshape_1"
  op: "Reshape"
  input: "electra/embeddings_2/MatMul"
  input: "electra/embeddings_2/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "electra/embeddings_3/Reshape"
  op: "Reshape"
  input: "InfeedQueue/dequeue:2"
  input: "electra/embeddings_3/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/one_hot/on_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/embeddings_3/one_hot/off_value"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_3/one_hot/depth"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "electra/embeddings_3/one_hot"
  op: "OneHot"
  input: "electra/embeddings_3/Reshape"
  input: "electra/embeddings_3/one_hot/depth"
  input: "electra/embeddings_3/one_hot/on_value"
  input: "electra/embeddings_3/one_hot/off_value"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "TI"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: -1
    }
  }
}
node {
  name: "electra/embeddings_3/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/MatMul"
  op: "MatMul"
  input: "electra/embeddings_3/one_hot"
  input: "electra/embeddings_3/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/embeddings_3/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings_3/Reshape_1"
  op: "Reshape"
  input: "electra/embeddings_3/MatMul"
  input: "electra/embeddings_3/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/add"
  op: "AddV2"
  input: "electra/embeddings_2/Reshape_1"
  input: "electra/embeddings_3/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 512
      }
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 512
      }
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/LessEqual"
  op: "LessEqual"
  input: "electra/embeddings_3/assert_less_equal/x"
  input: "electra/embeddings_3/assert_less_equal/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/Rank"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/range/start"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/range/delta"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/range"
  op: "Range"
  input: "electra/embeddings_3/assert_less_equal/range/start"
  input: "electra/embeddings_3/assert_less_equal/Rank"
  input: "electra/embeddings_3/assert_less_equal/range/delta"
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/All"
  op: "All"
  input: "electra/embeddings_3/assert_less_equal/LessEqual"
  input: "electra/embeddings_3/assert_less_equal/range"
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/Assert/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "Condition x <= y did not hold element-wise:"
      }
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/Assert/Const_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "x (electra/embeddings_3/assert_less_equal/x:0) = "
      }
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/Assert/Const_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "y (electra/embeddings_3/assert_less_equal/y:0) = "
      }
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/Assert/Assert/data_0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "Condition x <= y did not hold element-wise:"
      }
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/Assert/Assert/data_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "x (electra/embeddings_3/assert_less_equal/x:0) = "
      }
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/Assert/Assert/data_3"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "y (electra/embeddings_3/assert_less_equal/y:0) = "
      }
    }
  }
}
node {
  name: "electra/embeddings_3/assert_less_equal/Assert/Assert"
  op: "Assert"
  input: "electra/embeddings_3/assert_less_equal/All"
  input: "electra/embeddings_3/assert_less_equal/Assert/Assert/data_0"
  input: "electra/embeddings_3/assert_less_equal/Assert/Assert/data_1"
  input: "electra/embeddings_3/assert_less_equal/x"
  input: "electra/embeddings_3/assert_less_equal/Assert/Assert/data_3"
  input: "electra/embeddings_3/assert_less_equal/y"
  attr {
    key: "T"
    value {
      list {
        type: DT_STRING
        type: DT_STRING
        type: DT_INT32
        type: DT_STRING
        type: DT_INT32
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "summarize"
    value {
      i: 3
    }
  }
}
node {
  name: "electra/embeddings_3/Slice/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/Slice/ReadVariableOp/Enter"
  input: "^electra/embeddings_3/assert_less_equal/Assert/Assert"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/Slice/begin"
  op: "Const"
  input: "^electra/embeddings_3/assert_less_equal/Assert/Assert"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\000\000\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings_3/Slice/size"
  op: "Const"
  input: "^electra/embeddings_3/assert_less_equal/Assert/Assert"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\377\377\377\377"
      }
    }
  }
}
node {
  name: "electra/embeddings_3/Slice"
  op: "Slice"
  input: "electra/embeddings_3/Slice/ReadVariableOp"
  input: "electra/embeddings_3/Slice/begin"
  input: "electra/embeddings_3/Slice/size"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/Reshape_2/shape"
  op: "Const"
  input: "^electra/embeddings_3/assert_less_equal/Assert/Assert"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings_3/Reshape_2"
  op: "Reshape"
  input: "electra/embeddings_3/Slice"
  input: "electra/embeddings_3/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/add_1"
  op: "AddV2"
  input: "electra/embeddings_3/add"
  input: "electra/embeddings_3/Reshape_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_3/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma"
  input: "electra/embeddings_3/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_3/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta"
  input: "electra/embeddings_3/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/embeddings_3/add_1"
  input: "electra/embeddings_3/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/embeddings_3/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/embeddings_3/add_1"
  input: "electra/embeddings_3/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/embeddings_3/LayerNorm/moments/SquaredDifference"
  input: "electra/embeddings_3/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/embeddings_3/LayerNorm/moments/variance"
  input: "electra/embeddings_3/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/embeddings_3/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_3/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/embeddings_3/LayerNorm/batchnorm/Rsqrt"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/embeddings_3/add_1"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/embeddings_3/LayerNorm/moments/mean"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_3/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/embeddings_3/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul_1"
  input: "electra/embeddings_3/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/embeddings_3/dropout/Mul"
  op: "Mul"
  input: "electra/embeddings_3/LayerNorm/batchnorm/add_1"
  input: "electra/embeddings_3/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings_3/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/embeddings_3/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/embeddings_3/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/embeddings_3/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/embeddings_3/dropout/random_uniform/RandomUniform"
  input: "electra/embeddings_3/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/dropout/Cast"
  op: "Cast"
  input: "electra/embeddings_3/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings_3/dropout/Mul_1"
  op: "Mul"
  input: "electra/embeddings_3/dropout/Mul"
  input: "electra/embeddings_3/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape"
  op: "Reshape"
  input: "InfeedQueue/dequeue:1"
  input: "electra/encoder/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Cast"
  op: "Cast"
  input: "electra/encoder/Reshape"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/ones/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/ones/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/ones"
  op: "Fill"
  input: "electra/encoder/ones/shape_as_tensor"
  input: "electra/encoder/ones/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/mul"
  op: "Mul"
  input: "electra/encoder/ones"
  input: "electra/encoder/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\377\377\377\377\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_1"
  op: "Reshape"
  input: "electra/embeddings_3/dropout/Mul_1"
  input: "electra/encoder/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel"
  input: "electra/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias"
  input: "electra/encoder/layer_0/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/Reshape_1"
  input: "electra/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_0/attention/self/query/MatMul"
  input: "electra/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel"
  input: "electra/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias"
  input: "electra/encoder/layer_0/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/Reshape_1"
  input: "electra/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_0/attention/self/key/MatMul"
  input: "electra/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel"
  input: "electra/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias"
  input: "electra/encoder/layer_0/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/Reshape_1"
  input: "electra/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_0/attention/self/value/MatMul"
  input: "electra/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_0/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_0/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_0/attention/self/Reshape"
  input: "electra/encoder/layer_0/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_0/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_0/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_0/attention/self/Reshape_1"
  input: "electra/encoder/layer_0/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_0/attention/self/transpose"
  input: "electra/encoder/layer_0/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/self/MatMul"
  input: "electra/encoder/layer_0/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_0/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_0/attention/self/sub/x"
  input: "electra/encoder/layer_0/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/self/sub"
  input: "electra/encoder/layer_0/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_0/attention/self/Mul"
  input: "electra/encoder/layer_0/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_0/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/self/Softmax"
  input: "electra/encoder/layer_0/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_0/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_0/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_0/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_0/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/self/dropout/Mul"
  input: "electra/encoder/layer_0/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_0/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_0/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_0/attention/self/Reshape_2"
  input: "electra/encoder/layer_0/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_0/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_0/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_0/attention/self/MatMul_1"
  input: "electra/encoder/layer_0/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_0/attention/self/transpose_3"
  input: "electra/encoder/layer_0/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias"
  input: "electra/encoder/layer_0/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_0/attention/self/Reshape_3"
  input: "electra/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_0/attention/output/dense/MatMul"
  input: "electra/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_0/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_0/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_0/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_0/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_0/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/output/dropout/Mul"
  input: "electra/encoder/layer_0/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_0/attention/output/dropout/Mul_1"
  input: "electra/encoder/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_0/attention/output/add"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_0/attention/output/add"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/output/add"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias"
  input: "electra/encoder/layer_0/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_0/intermediate/dense/MatMul"
  input: "electra/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_0/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_0/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_0/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_0/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_0/intermediate/dense/add/x"
  input: "electra/encoder/layer_0/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_0/intermediate/dense/mul/x"
  input: "electra/encoder/layer_0/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_0/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel"
  input: "electra/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias"
  input: "electra/encoder/layer_0/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_0/intermediate/dense/mul_1"
  input: "electra/encoder/layer_0/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_0/output/dense/MatMul"
  input: "electra/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_0/output/dense/BiasAdd"
  input: "electra/encoder/layer_0/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_0/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_0/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_0/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_0/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/output/dropout/Mul"
  input: "electra/encoder/layer_0/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_0/output/dropout/Mul_1"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_0/output/add"
  input: "electra/encoder/layer_0/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_0/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_0/output/add"
  input: "electra/encoder/layer_0/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_0/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_0/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/output/add"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_0/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel"
  input: "electra/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias"
  input: "electra/encoder/layer_1/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_1/attention/self/query/MatMul"
  input: "electra/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel"
  input: "electra/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias"
  input: "electra/encoder/layer_1/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_1/attention/self/key/MatMul"
  input: "electra/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel"
  input: "electra/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias"
  input: "electra/encoder/layer_1/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_1/attention/self/value/MatMul"
  input: "electra/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_1/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_1/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_1/attention/self/Reshape"
  input: "electra/encoder/layer_1/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_1/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_1/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_1/attention/self/Reshape_1"
  input: "electra/encoder/layer_1/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_1/attention/self/transpose"
  input: "electra/encoder/layer_1/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/self/MatMul"
  input: "electra/encoder/layer_1/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_1/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_1/attention/self/sub/x"
  input: "electra/encoder/layer_1/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/self/sub"
  input: "electra/encoder/layer_1/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_1/attention/self/Mul"
  input: "electra/encoder/layer_1/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_1/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/self/Softmax"
  input: "electra/encoder/layer_1/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_1/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_1/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_1/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_1/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/self/dropout/Mul"
  input: "electra/encoder/layer_1/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_1/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_1/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_1/attention/self/Reshape_2"
  input: "electra/encoder/layer_1/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_1/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_1/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_1/attention/self/MatMul_1"
  input: "electra/encoder/layer_1/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_1/attention/self/transpose_3"
  input: "electra/encoder/layer_1/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias"
  input: "electra/encoder/layer_1/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_1/attention/self/Reshape_3"
  input: "electra/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_1/attention/output/dense/MatMul"
  input: "electra/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_1/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_1/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_1/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_1/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_1/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/output/dropout/Mul"
  input: "electra/encoder/layer_1/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_1/attention/output/dropout/Mul_1"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_1/attention/output/add"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_1/attention/output/add"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/output/add"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias"
  input: "electra/encoder/layer_1/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_1/intermediate/dense/MatMul"
  input: "electra/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_1/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_1/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_1/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_1/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_1/intermediate/dense/add/x"
  input: "electra/encoder/layer_1/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_1/intermediate/dense/mul/x"
  input: "electra/encoder/layer_1/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_1/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel"
  input: "electra/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias"
  input: "electra/encoder/layer_1/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_1/intermediate/dense/mul_1"
  input: "electra/encoder/layer_1/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_1/output/dense/MatMul"
  input: "electra/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_1/output/dense/BiasAdd"
  input: "electra/encoder/layer_1/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_1/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_1/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_1/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_1/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/output/dropout/Mul"
  input: "electra/encoder/layer_1/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_1/output/dropout/Mul_1"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_1/output/add"
  input: "electra/encoder/layer_1/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_1/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_1/output/add"
  input: "electra/encoder/layer_1/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_1/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_1/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/output/add"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_1/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel"
  input: "electra/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias"
  input: "electra/encoder/layer_2/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_2/attention/self/query/MatMul"
  input: "electra/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel"
  input: "electra/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias"
  input: "electra/encoder/layer_2/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_2/attention/self/key/MatMul"
  input: "electra/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel"
  input: "electra/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias"
  input: "electra/encoder/layer_2/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_2/attention/self/value/MatMul"
  input: "electra/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_2/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_2/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_2/attention/self/Reshape"
  input: "electra/encoder/layer_2/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_2/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_2/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_2/attention/self/Reshape_1"
  input: "electra/encoder/layer_2/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_2/attention/self/transpose"
  input: "electra/encoder/layer_2/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/self/MatMul"
  input: "electra/encoder/layer_2/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_2/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_2/attention/self/sub/x"
  input: "electra/encoder/layer_2/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/self/sub"
  input: "electra/encoder/layer_2/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_2/attention/self/Mul"
  input: "electra/encoder/layer_2/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_2/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/self/Softmax"
  input: "electra/encoder/layer_2/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_2/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_2/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_2/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_2/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/self/dropout/Mul"
  input: "electra/encoder/layer_2/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_2/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_2/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_2/attention/self/Reshape_2"
  input: "electra/encoder/layer_2/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_2/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_2/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_2/attention/self/MatMul_1"
  input: "electra/encoder/layer_2/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_2/attention/self/transpose_3"
  input: "electra/encoder/layer_2/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias"
  input: "electra/encoder/layer_2/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_2/attention/self/Reshape_3"
  input: "electra/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_2/attention/output/dense/MatMul"
  input: "electra/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_2/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_2/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_2/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_2/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_2/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/output/dropout/Mul"
  input: "electra/encoder/layer_2/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_2/attention/output/dropout/Mul_1"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_2/attention/output/add"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_2/attention/output/add"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/output/add"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias"
  input: "electra/encoder/layer_2/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_2/intermediate/dense/MatMul"
  input: "electra/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_2/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_2/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_2/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_2/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_2/intermediate/dense/add/x"
  input: "electra/encoder/layer_2/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_2/intermediate/dense/mul/x"
  input: "electra/encoder/layer_2/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_2/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel"
  input: "electra/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias"
  input: "electra/encoder/layer_2/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_2/intermediate/dense/mul_1"
  input: "electra/encoder/layer_2/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_2/output/dense/MatMul"
  input: "electra/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_2/output/dense/BiasAdd"
  input: "electra/encoder/layer_2/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_2/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_2/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_2/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_2/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/output/dropout/Mul"
  input: "electra/encoder/layer_2/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_2/output/dropout/Mul_1"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_2/output/add"
  input: "electra/encoder/layer_2/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_2/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_2/output/add"
  input: "electra/encoder/layer_2/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_2/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_2/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/output/add"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_2/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel"
  input: "electra/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias"
  input: "electra/encoder/layer_3/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_3/attention/self/query/MatMul"
  input: "electra/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel"
  input: "electra/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias"
  input: "electra/encoder/layer_3/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_3/attention/self/key/MatMul"
  input: "electra/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel"
  input: "electra/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias"
  input: "electra/encoder/layer_3/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_3/attention/self/value/MatMul"
  input: "electra/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_3/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_3/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_3/attention/self/Reshape"
  input: "electra/encoder/layer_3/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_3/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_3/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_3/attention/self/Reshape_1"
  input: "electra/encoder/layer_3/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_3/attention/self/transpose"
  input: "electra/encoder/layer_3/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/self/MatMul"
  input: "electra/encoder/layer_3/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_3/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_3/attention/self/sub/x"
  input: "electra/encoder/layer_3/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/self/sub"
  input: "electra/encoder/layer_3/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_3/attention/self/Mul"
  input: "electra/encoder/layer_3/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_3/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/self/Softmax"
  input: "electra/encoder/layer_3/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_3/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_3/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_3/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_3/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/self/dropout/Mul"
  input: "electra/encoder/layer_3/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_3/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_3/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_3/attention/self/Reshape_2"
  input: "electra/encoder/layer_3/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_3/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_3/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_3/attention/self/MatMul_1"
  input: "electra/encoder/layer_3/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_3/attention/self/transpose_3"
  input: "electra/encoder/layer_3/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias"
  input: "electra/encoder/layer_3/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_3/attention/self/Reshape_3"
  input: "electra/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_3/attention/output/dense/MatMul"
  input: "electra/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_3/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_3/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_3/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_3/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_3/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/output/dropout/Mul"
  input: "electra/encoder/layer_3/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_3/attention/output/dropout/Mul_1"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_3/attention/output/add"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_3/attention/output/add"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/output/add"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias"
  input: "electra/encoder/layer_3/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_3/intermediate/dense/MatMul"
  input: "electra/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_3/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_3/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_3/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_3/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_3/intermediate/dense/add/x"
  input: "electra/encoder/layer_3/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_3/intermediate/dense/mul/x"
  input: "electra/encoder/layer_3/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_3/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel"
  input: "electra/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias"
  input: "electra/encoder/layer_3/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_3/intermediate/dense/mul_1"
  input: "electra/encoder/layer_3/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_3/output/dense/MatMul"
  input: "electra/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_3/output/dense/BiasAdd"
  input: "electra/encoder/layer_3/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_3/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_3/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_3/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_3/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/output/dropout/Mul"
  input: "electra/encoder/layer_3/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_3/output/dropout/Mul_1"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_3/output/add"
  input: "electra/encoder/layer_3/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_3/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_3/output/add"
  input: "electra/encoder/layer_3/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_3/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_3/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/output/add"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_3/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel"
  input: "electra/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias"
  input: "electra/encoder/layer_4/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_4/attention/self/query/MatMul"
  input: "electra/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel"
  input: "electra/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias"
  input: "electra/encoder/layer_4/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_4/attention/self/key/MatMul"
  input: "electra/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel"
  input: "electra/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias"
  input: "electra/encoder/layer_4/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_4/attention/self/value/MatMul"
  input: "electra/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_4/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_4/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_4/attention/self/Reshape"
  input: "electra/encoder/layer_4/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_4/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_4/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_4/attention/self/Reshape_1"
  input: "electra/encoder/layer_4/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_4/attention/self/transpose"
  input: "electra/encoder/layer_4/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/self/MatMul"
  input: "electra/encoder/layer_4/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_4/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_4/attention/self/sub/x"
  input: "electra/encoder/layer_4/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/self/sub"
  input: "electra/encoder/layer_4/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_4/attention/self/Mul"
  input: "electra/encoder/layer_4/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_4/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/self/Softmax"
  input: "electra/encoder/layer_4/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_4/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_4/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_4/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_4/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/self/dropout/Mul"
  input: "electra/encoder/layer_4/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_4/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_4/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_4/attention/self/Reshape_2"
  input: "electra/encoder/layer_4/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_4/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_4/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_4/attention/self/MatMul_1"
  input: "electra/encoder/layer_4/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_4/attention/self/transpose_3"
  input: "electra/encoder/layer_4/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias"
  input: "electra/encoder/layer_4/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_4/attention/self/Reshape_3"
  input: "electra/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_4/attention/output/dense/MatMul"
  input: "electra/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_4/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_4/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_4/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_4/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_4/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/output/dropout/Mul"
  input: "electra/encoder/layer_4/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_4/attention/output/dropout/Mul_1"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_4/attention/output/add"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_4/attention/output/add"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/output/add"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias"
  input: "electra/encoder/layer_4/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_4/intermediate/dense/MatMul"
  input: "electra/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_4/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_4/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_4/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_4/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_4/intermediate/dense/add/x"
  input: "electra/encoder/layer_4/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_4/intermediate/dense/mul/x"
  input: "electra/encoder/layer_4/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_4/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel"
  input: "electra/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias"
  input: "electra/encoder/layer_4/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_4/intermediate/dense/mul_1"
  input: "electra/encoder/layer_4/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_4/output/dense/MatMul"
  input: "electra/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_4/output/dense/BiasAdd"
  input: "electra/encoder/layer_4/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_4/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_4/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_4/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_4/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/output/dropout/Mul"
  input: "electra/encoder/layer_4/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_4/output/dropout/Mul_1"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_4/output/add"
  input: "electra/encoder/layer_4/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_4/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_4/output/add"
  input: "electra/encoder/layer_4/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_4/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_4/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/output/add"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_4/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel"
  input: "electra/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias"
  input: "electra/encoder/layer_5/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_5/attention/self/query/MatMul"
  input: "electra/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel"
  input: "electra/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias"
  input: "electra/encoder/layer_5/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_5/attention/self/key/MatMul"
  input: "electra/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel"
  input: "electra/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias"
  input: "electra/encoder/layer_5/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_5/attention/self/value/MatMul"
  input: "electra/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_5/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_5/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_5/attention/self/Reshape"
  input: "electra/encoder/layer_5/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_5/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_5/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_5/attention/self/Reshape_1"
  input: "electra/encoder/layer_5/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_5/attention/self/transpose"
  input: "electra/encoder/layer_5/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/self/MatMul"
  input: "electra/encoder/layer_5/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_5/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_5/attention/self/sub/x"
  input: "electra/encoder/layer_5/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/self/sub"
  input: "electra/encoder/layer_5/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_5/attention/self/Mul"
  input: "electra/encoder/layer_5/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_5/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/self/Softmax"
  input: "electra/encoder/layer_5/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_5/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_5/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_5/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_5/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/self/dropout/Mul"
  input: "electra/encoder/layer_5/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_5/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_5/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_5/attention/self/Reshape_2"
  input: "electra/encoder/layer_5/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_5/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_5/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_5/attention/self/MatMul_1"
  input: "electra/encoder/layer_5/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_5/attention/self/transpose_3"
  input: "electra/encoder/layer_5/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias"
  input: "electra/encoder/layer_5/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_5/attention/self/Reshape_3"
  input: "electra/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_5/attention/output/dense/MatMul"
  input: "electra/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_5/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_5/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_5/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_5/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_5/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/output/dropout/Mul"
  input: "electra/encoder/layer_5/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_5/attention/output/dropout/Mul_1"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_5/attention/output/add"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_5/attention/output/add"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/output/add"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias"
  input: "electra/encoder/layer_5/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_5/intermediate/dense/MatMul"
  input: "electra/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_5/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_5/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_5/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_5/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_5/intermediate/dense/add/x"
  input: "electra/encoder/layer_5/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_5/intermediate/dense/mul/x"
  input: "electra/encoder/layer_5/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_5/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel"
  input: "electra/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias"
  input: "electra/encoder/layer_5/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_5/intermediate/dense/mul_1"
  input: "electra/encoder/layer_5/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_5/output/dense/MatMul"
  input: "electra/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_5/output/dense/BiasAdd"
  input: "electra/encoder/layer_5/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_5/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_5/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_5/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_5/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/output/dropout/Mul"
  input: "electra/encoder/layer_5/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_5/output/dropout/Mul_1"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_5/output/add"
  input: "electra/encoder/layer_5/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_5/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_5/output/add"
  input: "electra/encoder/layer_5/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_5/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_5/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/output/add"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_5/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel"
  input: "electra/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias"
  input: "electra/encoder/layer_6/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_6/attention/self/query/MatMul"
  input: "electra/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel"
  input: "electra/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias"
  input: "electra/encoder/layer_6/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_6/attention/self/key/MatMul"
  input: "electra/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel"
  input: "electra/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias"
  input: "electra/encoder/layer_6/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_6/attention/self/value/MatMul"
  input: "electra/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_6/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_6/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_6/attention/self/Reshape"
  input: "electra/encoder/layer_6/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_6/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_6/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_6/attention/self/Reshape_1"
  input: "electra/encoder/layer_6/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_6/attention/self/transpose"
  input: "electra/encoder/layer_6/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/self/MatMul"
  input: "electra/encoder/layer_6/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_6/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_6/attention/self/sub/x"
  input: "electra/encoder/layer_6/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/self/sub"
  input: "electra/encoder/layer_6/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_6/attention/self/Mul"
  input: "electra/encoder/layer_6/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_6/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/self/Softmax"
  input: "electra/encoder/layer_6/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_6/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_6/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_6/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_6/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/self/dropout/Mul"
  input: "electra/encoder/layer_6/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_6/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_6/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_6/attention/self/Reshape_2"
  input: "electra/encoder/layer_6/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_6/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_6/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_6/attention/self/MatMul_1"
  input: "electra/encoder/layer_6/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_6/attention/self/transpose_3"
  input: "electra/encoder/layer_6/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias"
  input: "electra/encoder/layer_6/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_6/attention/self/Reshape_3"
  input: "electra/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_6/attention/output/dense/MatMul"
  input: "electra/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_6/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_6/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_6/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_6/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_6/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/output/dropout/Mul"
  input: "electra/encoder/layer_6/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_6/attention/output/dropout/Mul_1"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_6/attention/output/add"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_6/attention/output/add"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/output/add"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias"
  input: "electra/encoder/layer_6/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_6/intermediate/dense/MatMul"
  input: "electra/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_6/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_6/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_6/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_6/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_6/intermediate/dense/add/x"
  input: "electra/encoder/layer_6/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_6/intermediate/dense/mul/x"
  input: "electra/encoder/layer_6/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_6/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel"
  input: "electra/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias"
  input: "electra/encoder/layer_6/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_6/intermediate/dense/mul_1"
  input: "electra/encoder/layer_6/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_6/output/dense/MatMul"
  input: "electra/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_6/output/dense/BiasAdd"
  input: "electra/encoder/layer_6/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_6/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_6/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_6/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_6/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/output/dropout/Mul"
  input: "electra/encoder/layer_6/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_6/output/dropout/Mul_1"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_6/output/add"
  input: "electra/encoder/layer_6/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_6/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_6/output/add"
  input: "electra/encoder/layer_6/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_6/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_6/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/output/add"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_6/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel"
  input: "electra/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias"
  input: "electra/encoder/layer_7/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_7/attention/self/query/MatMul"
  input: "electra/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel"
  input: "electra/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias"
  input: "electra/encoder/layer_7/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_7/attention/self/key/MatMul"
  input: "electra/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel"
  input: "electra/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias"
  input: "electra/encoder/layer_7/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_7/attention/self/value/MatMul"
  input: "electra/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_7/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_7/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_7/attention/self/Reshape"
  input: "electra/encoder/layer_7/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_7/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_7/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_7/attention/self/Reshape_1"
  input: "electra/encoder/layer_7/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_7/attention/self/transpose"
  input: "electra/encoder/layer_7/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/self/MatMul"
  input: "electra/encoder/layer_7/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_7/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_7/attention/self/sub/x"
  input: "electra/encoder/layer_7/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/self/sub"
  input: "electra/encoder/layer_7/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_7/attention/self/Mul"
  input: "electra/encoder/layer_7/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_7/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/self/Softmax"
  input: "electra/encoder/layer_7/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_7/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_7/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_7/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_7/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/self/dropout/Mul"
  input: "electra/encoder/layer_7/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_7/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_7/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_7/attention/self/Reshape_2"
  input: "electra/encoder/layer_7/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_7/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_7/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_7/attention/self/MatMul_1"
  input: "electra/encoder/layer_7/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_7/attention/self/transpose_3"
  input: "electra/encoder/layer_7/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias"
  input: "electra/encoder/layer_7/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_7/attention/self/Reshape_3"
  input: "electra/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_7/attention/output/dense/MatMul"
  input: "electra/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_7/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_7/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_7/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_7/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_7/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/output/dropout/Mul"
  input: "electra/encoder/layer_7/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_7/attention/output/dropout/Mul_1"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_7/attention/output/add"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_7/attention/output/add"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/output/add"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias"
  input: "electra/encoder/layer_7/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_7/intermediate/dense/MatMul"
  input: "electra/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_7/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_7/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_7/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_7/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_7/intermediate/dense/add/x"
  input: "electra/encoder/layer_7/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_7/intermediate/dense/mul/x"
  input: "electra/encoder/layer_7/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_7/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel"
  input: "electra/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias"
  input: "electra/encoder/layer_7/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_7/intermediate/dense/mul_1"
  input: "electra/encoder/layer_7/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_7/output/dense/MatMul"
  input: "electra/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_7/output/dense/BiasAdd"
  input: "electra/encoder/layer_7/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_7/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_7/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_7/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_7/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/output/dropout/Mul"
  input: "electra/encoder/layer_7/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_7/output/dropout/Mul_1"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_7/output/add"
  input: "electra/encoder/layer_7/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_7/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_7/output/add"
  input: "electra/encoder/layer_7/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_7/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_7/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/output/add"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_7/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel"
  input: "electra/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias"
  input: "electra/encoder/layer_8/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_8/attention/self/query/MatMul"
  input: "electra/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel"
  input: "electra/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias"
  input: "electra/encoder/layer_8/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_8/attention/self/key/MatMul"
  input: "electra/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel"
  input: "electra/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias"
  input: "electra/encoder/layer_8/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_8/attention/self/value/MatMul"
  input: "electra/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_8/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_8/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_8/attention/self/Reshape"
  input: "electra/encoder/layer_8/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_8/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_8/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_8/attention/self/Reshape_1"
  input: "electra/encoder/layer_8/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_8/attention/self/transpose"
  input: "electra/encoder/layer_8/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/self/MatMul"
  input: "electra/encoder/layer_8/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_8/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_8/attention/self/sub/x"
  input: "electra/encoder/layer_8/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/self/sub"
  input: "electra/encoder/layer_8/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_8/attention/self/Mul"
  input: "electra/encoder/layer_8/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_8/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/self/Softmax"
  input: "electra/encoder/layer_8/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_8/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_8/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_8/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_8/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/self/dropout/Mul"
  input: "electra/encoder/layer_8/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_8/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_8/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_8/attention/self/Reshape_2"
  input: "electra/encoder/layer_8/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_8/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_8/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_8/attention/self/MatMul_1"
  input: "electra/encoder/layer_8/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_8/attention/self/transpose_3"
  input: "electra/encoder/layer_8/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias"
  input: "electra/encoder/layer_8/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_8/attention/self/Reshape_3"
  input: "electra/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_8/attention/output/dense/MatMul"
  input: "electra/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_8/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_8/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_8/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_8/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_8/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/output/dropout/Mul"
  input: "electra/encoder/layer_8/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_8/attention/output/dropout/Mul_1"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_8/attention/output/add"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_8/attention/output/add"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/output/add"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias"
  input: "electra/encoder/layer_8/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_8/intermediate/dense/MatMul"
  input: "electra/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_8/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_8/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_8/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_8/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_8/intermediate/dense/add/x"
  input: "electra/encoder/layer_8/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_8/intermediate/dense/mul/x"
  input: "electra/encoder/layer_8/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_8/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel"
  input: "electra/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias"
  input: "electra/encoder/layer_8/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_8/intermediate/dense/mul_1"
  input: "electra/encoder/layer_8/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_8/output/dense/MatMul"
  input: "electra/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_8/output/dense/BiasAdd"
  input: "electra/encoder/layer_8/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_8/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_8/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_8/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_8/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/output/dropout/Mul"
  input: "electra/encoder/layer_8/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_8/output/dropout/Mul_1"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_8/output/add"
  input: "electra/encoder/layer_8/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_8/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_8/output/add"
  input: "electra/encoder/layer_8/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_8/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_8/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/output/add"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_8/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel"
  input: "electra/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias"
  input: "electra/encoder/layer_9/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_9/attention/self/query/MatMul"
  input: "electra/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel"
  input: "electra/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias"
  input: "electra/encoder/layer_9/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_9/attention/self/key/MatMul"
  input: "electra/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel"
  input: "electra/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias"
  input: "electra/encoder/layer_9/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_9/attention/self/value/MatMul"
  input: "electra/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_9/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_9/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_9/attention/self/Reshape"
  input: "electra/encoder/layer_9/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_9/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_9/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_9/attention/self/Reshape_1"
  input: "electra/encoder/layer_9/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_9/attention/self/transpose"
  input: "electra/encoder/layer_9/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/self/MatMul"
  input: "electra/encoder/layer_9/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_9/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_9/attention/self/sub/x"
  input: "electra/encoder/layer_9/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/self/sub"
  input: "electra/encoder/layer_9/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_9/attention/self/Mul"
  input: "electra/encoder/layer_9/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_9/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/self/Softmax"
  input: "electra/encoder/layer_9/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_9/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_9/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_9/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_9/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/self/dropout/Mul"
  input: "electra/encoder/layer_9/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_9/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_9/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_9/attention/self/Reshape_2"
  input: "electra/encoder/layer_9/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_9/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_9/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_9/attention/self/MatMul_1"
  input: "electra/encoder/layer_9/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_9/attention/self/transpose_3"
  input: "electra/encoder/layer_9/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias"
  input: "electra/encoder/layer_9/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_9/attention/self/Reshape_3"
  input: "electra/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_9/attention/output/dense/MatMul"
  input: "electra/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_9/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_9/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_9/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_9/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_9/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/output/dropout/Mul"
  input: "electra/encoder/layer_9/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_9/attention/output/dropout/Mul_1"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_9/attention/output/add"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_9/attention/output/add"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/output/add"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias"
  input: "electra/encoder/layer_9/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_9/intermediate/dense/MatMul"
  input: "electra/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_9/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_9/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_9/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_9/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_9/intermediate/dense/add/x"
  input: "electra/encoder/layer_9/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_9/intermediate/dense/mul/x"
  input: "electra/encoder/layer_9/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_9/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel"
  input: "electra/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias"
  input: "electra/encoder/layer_9/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_9/intermediate/dense/mul_1"
  input: "electra/encoder/layer_9/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_9/output/dense/MatMul"
  input: "electra/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_9/output/dense/BiasAdd"
  input: "electra/encoder/layer_9/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_9/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_9/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_9/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_9/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/output/dropout/Mul"
  input: "electra/encoder/layer_9/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_9/output/dropout/Mul_1"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_9/output/add"
  input: "electra/encoder/layer_9/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_9/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_9/output/add"
  input: "electra/encoder/layer_9/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_9/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_9/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/output/add"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_9/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel"
  input: "electra/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias"
  input: "electra/encoder/layer_10/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_10/attention/self/query/MatMul"
  input: "electra/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel"
  input: "electra/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias"
  input: "electra/encoder/layer_10/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_10/attention/self/key/MatMul"
  input: "electra/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel"
  input: "electra/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias"
  input: "electra/encoder/layer_10/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_10/attention/self/value/MatMul"
  input: "electra/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_10/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_10/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_10/attention/self/Reshape"
  input: "electra/encoder/layer_10/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_10/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_10/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_10/attention/self/Reshape_1"
  input: "electra/encoder/layer_10/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_10/attention/self/transpose"
  input: "electra/encoder/layer_10/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/self/MatMul"
  input: "electra/encoder/layer_10/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_10/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_10/attention/self/sub/x"
  input: "electra/encoder/layer_10/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/self/sub"
  input: "electra/encoder/layer_10/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_10/attention/self/Mul"
  input: "electra/encoder/layer_10/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_10/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/self/Softmax"
  input: "electra/encoder/layer_10/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_10/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_10/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_10/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_10/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/self/dropout/Mul"
  input: "electra/encoder/layer_10/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_10/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_10/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_10/attention/self/Reshape_2"
  input: "electra/encoder/layer_10/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_10/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_10/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_10/attention/self/MatMul_1"
  input: "electra/encoder/layer_10/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_10/attention/self/transpose_3"
  input: "electra/encoder/layer_10/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias"
  input: "electra/encoder/layer_10/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_10/attention/self/Reshape_3"
  input: "electra/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_10/attention/output/dense/MatMul"
  input: "electra/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_10/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_10/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_10/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_10/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_10/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/output/dropout/Mul"
  input: "electra/encoder/layer_10/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_10/attention/output/dropout/Mul_1"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_10/attention/output/add"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_10/attention/output/add"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/output/add"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias"
  input: "electra/encoder/layer_10/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_10/intermediate/dense/MatMul"
  input: "electra/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_10/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_10/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_10/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_10/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_10/intermediate/dense/add/x"
  input: "electra/encoder/layer_10/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_10/intermediate/dense/mul/x"
  input: "electra/encoder/layer_10/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_10/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel"
  input: "electra/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias"
  input: "electra/encoder/layer_10/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_10/intermediate/dense/mul_1"
  input: "electra/encoder/layer_10/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_10/output/dense/MatMul"
  input: "electra/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_10/output/dense/BiasAdd"
  input: "electra/encoder/layer_10/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_10/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_10/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_10/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_10/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/output/dropout/Mul"
  input: "electra/encoder/layer_10/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_10/output/dropout/Mul_1"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_10/output/add"
  input: "electra/encoder/layer_10/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_10/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_10/output/add"
  input: "electra/encoder/layer_10/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_10/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_10/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/output/add"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_10/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/query/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel"
  input: "electra/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/query/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias"
  input: "electra/encoder/layer_11/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/query/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/query/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_11/attention/self/query/MatMul"
  input: "electra/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/key/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel"
  input: "electra/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/key/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias"
  input: "electra/encoder/layer_11/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/key/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/key/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_11/attention/self/key/MatMul"
  input: "electra/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/value/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel"
  input: "electra/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/value/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias"
  input: "electra/encoder/layer_11/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/value/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/value/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_11/attention/self/value/MatMul"
  input: "electra/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/Reshape"
  op: "Reshape"
  input: "electra/encoder/layer_11/attention/self/query/BiasAdd"
  input: "electra/encoder/layer_11/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/transpose/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/transpose"
  op: "Transpose"
  input: "electra/encoder/layer_11/attention/self/Reshape"
  input: "electra/encoder/layer_11/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/Reshape_1"
  op: "Reshape"
  input: "electra/encoder/layer_11/attention/self/key/BiasAdd"
  input: "electra/encoder/layer_11/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/transpose_1/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/transpose_1"
  op: "Transpose"
  input: "electra/encoder/layer_11/attention/self/Reshape_1"
  input: "electra/encoder/layer_11/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/MatMul"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_11/attention/self/transpose"
  input: "electra/encoder/layer_11/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/Mul/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/Mul"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/self/MatMul"
  input: "electra/encoder/layer_11/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "electra/encoder/mul"
  input: "electra/encoder/layer_11/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/sub/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/sub"
  op: "Sub"
  input: "electra/encoder/layer_11/attention/self/sub/x"
  input: "electra/encoder/layer_11/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/mul_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/self/sub"
  input: "electra/encoder/layer_11/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/add"
  op: "AddV2"
  input: "electra/encoder/layer_11/attention/self/Mul"
  input: "electra/encoder/layer_11/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/Softmax"
  op: "Softmax"
  input: "electra/encoder/layer_11/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/self/Softmax"
  input: "electra/encoder/layer_11/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_11/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_11/attention/self/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_11/attention/self/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_11/attention/self/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/self/dropout/Mul"
  input: "electra/encoder/layer_11/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_11/attention/self/value/BiasAdd"
  input: "electra/encoder/layer_11/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/transpose_2/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/transpose_2"
  op: "Transpose"
  input: "electra/encoder/layer_11/attention/self/Reshape_2"
  input: "electra/encoder/layer_11/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_11/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_11/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/transpose_3/perm"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/transpose_3"
  op: "Transpose"
  input: "electra/encoder/layer_11/attention/self/MatMul_1"
  input: "electra/encoder/layer_11/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_11/attention/self/transpose_3"
  input: "electra/encoder/layer_11/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias"
  input: "electra/encoder/layer_11/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_11/attention/self/Reshape_3"
  input: "electra/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_11/attention/output/dense/MatMul"
  input: "electra/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/output/dense/BiasAdd"
  input: "electra/encoder/layer_11/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_11/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_11/attention/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_11/attention/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_11/attention/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/output/dropout/Mul"
  input: "electra/encoder/layer_11/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_11/attention/output/dropout/Mul_1"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_11/attention/output/add"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_11/attention/output/add"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/output/add"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/intermediate/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/intermediate/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias"
  input: "electra/encoder/layer_11/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/intermediate/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/intermediate/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_11/intermediate/dense/MatMul"
  input: "electra/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "electra/encoder/layer_11/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/truediv"
  op: "RealDiv"
  input: "electra/encoder/layer_11/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_11/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/Erf"
  op: "Erf"
  input: "electra/encoder/layer_11/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/add"
  op: "AddV2"
  input: "electra/encoder/layer_11/intermediate/dense/add/x"
  input: "electra/encoder/layer_11/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/mul"
  op: "Mul"
  input: "electra/encoder/layer_11/intermediate/dense/mul/x"
  input: "electra/encoder/layer_11/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/intermediate/dense/BiasAdd"
  input: "electra/encoder/layer_11/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/dense/kernel"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel"
  input: "electra/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/dense/bias"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias"
  input: "electra/encoder/layer_11/output/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/MatMul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/MatMul"
  op: "MatMul"
  input: "electra/encoder/layer_11/intermediate/dense/mul_1"
  input: "electra/encoder/layer_11/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "electra/encoder/layer_11/output/dense/MatMul"
  input: "electra/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dropout/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.1111111640930176
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dropout/Mul"
  op: "Mul"
  input: "electra/encoder/layer_11/output/dense/BiasAdd"
  input: "electra/encoder/layer_11/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dropout/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "electra/encoder/layer_11/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dropout/GreaterEqual/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dropout/GreaterEqual"
  op: "GreaterEqual"
  input: "electra/encoder/layer_11/output/dropout/random_uniform/RandomUniform"
  input: "electra/encoder/layer_11/output/dropout/GreaterEqual/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dropout/Cast"
  op: "Cast"
  input: "electra/encoder/layer_11/output/dropout/GreaterEqual"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dropout/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/output/dropout/Mul"
  input: "electra/encoder/layer_11/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/add"
  op: "AddV2"
  input: "electra/encoder/layer_11/output/dropout/Mul_1"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/LayerNorm/gamma"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/LayerNorm/beta"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "electra/encoder/layer_11/output/add"
  input: "electra/encoder/layer_11/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "electra/encoder/layer_11/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "electra/encoder/layer_11/output/add"
  input: "electra/encoder/layer_11/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference"
  input: "electra/encoder/layer_11/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/add"
  op: "AddV2"
  input: "electra/encoder/layer_11/output/LayerNorm/moments/variance"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/output/add"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "electra/encoder/layer_11/output/LayerNorm/moments/mean"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/LayerNorm/beta"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/batchnorm/add_1"
  op: "AddV2"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_1"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/stack"
  op: "Pack"
  input: "electra/encoder/layer_0/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_1/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_2/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_3/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_4/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_5/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_6/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_7/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_8/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_9/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_10/attention/self/dropout/Mul_1"
  input: "electra/encoder/layer_11/attention/self/dropout/Mul_1"
  attr {
    key: "N"
    value {
      i: 12
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 12
          }
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/Reshape_2/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_2"
  op: "Reshape"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_3/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_3"
  op: "Reshape"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_4/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_4"
  op: "Reshape"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_4/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_5/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_5"
  op: "Reshape"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_5/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_6/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_6"
  op: "Reshape"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_6/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_7/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_7"
  op: "Reshape"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_7/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_8/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_8"
  op: "Reshape"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_8/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_9/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_9"
  op: "Reshape"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_9/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_10/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_10"
  op: "Reshape"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_10/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_11/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_11"
  op: "Reshape"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_11/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_12/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_12"
  op: "Reshape"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_12/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/Reshape_13/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/Reshape_13"
  op: "Reshape"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/add_1"
  input: "electra/encoder/Reshape_13/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/encoder/stack_1"
  op: "Pack"
  input: "electra/encoder/Reshape_2"
  input: "electra/encoder/Reshape_3"
  input: "electra/encoder/Reshape_4"
  input: "electra/encoder/Reshape_5"
  input: "electra/encoder/Reshape_6"
  input: "electra/encoder/Reshape_7"
  input: "electra/encoder/Reshape_8"
  input: "electra/encoder/Reshape_9"
  input: "electra/encoder/Reshape_10"
  input: "electra/encoder/Reshape_11"
  input: "electra/encoder/Reshape_12"
  input: "electra/encoder/Reshape_13"
  attr {
    key: "N"
    value {
      i: 12
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 12
          }
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
}
node {
  name: "electra/encoder/strided_slice/stack"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "electra/encoder/strided_slice/stack_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "electra/encoder/strided_slice/stack_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "electra/encoder/strided_slice"
  op: "StridedSlice"
  input: "electra/encoder/stack_1"
  input: "electra/encoder/strided_slice/stack"
  input: "electra/encoder/strided_slice/stack_1"
  input: "electra/encoder/strided_slice/stack_2"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 1
    }
  }
}
node {
  name: "electra/encoder/strided_slice_1/stack"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\000\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/strided_slice_1/stack_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/strided_slice_1/stack_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/strided_slice_1"
  op: "StridedSlice"
  input: "electra/encoder/strided_slice"
  input: "electra/encoder/strided_slice_1/stack"
  input: "electra/encoder/strided_slice_1/stack_1"
  input: "electra/encoder/strided_slice_1/stack_2"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 1
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 1
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 2
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "discriminator_predictions/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "discriminator_predictions/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "discriminator_predictions/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/Initializer/truncated_normal"
  op: "AddV2"
  input: "discriminator_predictions/dense/kernel/Initializer/truncated_normal/mul"
  input: "discriminator_predictions/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense/kernel"
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/kernel"
  input: "discriminator_predictions/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense/bias"
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/bias"
  input: "discriminator_predictions/dense/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/Tensordot/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "discriminator_predictions/dense/Tensordot/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/Tensordot/Reshape"
  op: "Reshape"
  input: "electra/encoder/strided_slice"
  input: "discriminator_predictions/dense/Tensordot/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/dense/Tensordot/MatMul"
  op: "MatMul"
  input: "discriminator_predictions/dense/Tensordot/Reshape"
  input: "discriminator_predictions/dense/Tensordot/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "discriminator_predictions/dense/Tensordot/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/Tensordot"
  op: "Reshape"
  input: "discriminator_predictions/dense/Tensordot/MatMul"
  input: "discriminator_predictions/dense/Tensordot/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/dense/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "discriminator_predictions/dense/BiasAdd"
  op: "BiasAdd"
  input: "discriminator_predictions/dense/Tensordot"
  input: "discriminator_predictions/dense/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "discriminator_predictions/dense/Sqrt/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/Sqrt"
  op: "Sqrt"
  input: "discriminator_predictions/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/dense/truediv"
  op: "RealDiv"
  input: "discriminator_predictions/dense/BiasAdd"
  input: "discriminator_predictions/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/dense/Erf"
  op: "Erf"
  input: "discriminator_predictions/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/dense/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/add"
  op: "AddV2"
  input: "discriminator_predictions/dense/add/x"
  input: "discriminator_predictions/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/dense/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/mul"
  op: "Mul"
  input: "discriminator_predictions/dense/mul/x"
  input: "discriminator_predictions/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/dense/mul_1"
  op: "Mul"
  input: "discriminator_predictions/dense/BiasAdd"
  input: "discriminator_predictions/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/min"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -0.08833085745573044
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/max"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.08833085745573044
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/sub"
  op: "Sub"
  input: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/max"
  input: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/mul"
  op: "Mul"
  input: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/RandomUniform"
  input: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform"
  op: "AddV2"
  input: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/mul"
  input: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 1
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense_1/kernel"
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/kernel"
  input: "discriminator_predictions/dense_1/kernel/Initializer/random_uniform"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 1
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense_1/bias"
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/bias"
  input: "discriminator_predictions/dense_1/bias/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/Tensordot/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/Tensordot/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense_1/kernel"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/Tensordot/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/Tensordot/Reshape"
  op: "Reshape"
  input: "discriminator_predictions/dense/mul_1"
  input: "discriminator_predictions/dense_1/Tensordot/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/Tensordot/MatMul"
  op: "MatMul"
  input: "discriminator_predictions/dense_1/Tensordot/Reshape"
  input: "discriminator_predictions/dense_1/Tensordot/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/Tensordot/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/Tensordot"
  op: "Reshape"
  input: "discriminator_predictions/dense_1/Tensordot/MatMul"
  input: "discriminator_predictions/dense_1/Tensordot/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/BiasAdd/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/BiasAdd/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense_1/bias"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/BiasAdd"
  op: "BiasAdd"
  input: "discriminator_predictions/dense_1/Tensordot"
  input: "discriminator_predictions/dense_1/BiasAdd/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "discriminator_predictions/Squeeze"
  op: "Squeeze"
  input: "discriminator_predictions/dense_1/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "squeeze_dims"
    value {
      list {
        i: -1
      }
    }
  }
}
node {
  name: "discriminator_predictions/Cast"
  op: "Cast"
  input: "InfeedQueue/dequeue:1"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/Cast_1"
  op: "Cast"
  input: "mul_18"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss/zeros_like/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss/zeros_like/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss/zeros_like"
  op: "Fill"
  input: "discriminator_predictions/logistic_loss/zeros_like/shape_as_tensor"
  input: "discriminator_predictions/logistic_loss/zeros_like/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss/GreaterEqual"
  op: "GreaterEqual"
  input: "discriminator_predictions/Squeeze"
  input: "discriminator_predictions/logistic_loss/zeros_like"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss/Select"
  op: "Select"
  input: "discriminator_predictions/logistic_loss/GreaterEqual"
  input: "discriminator_predictions/Squeeze"
  input: "discriminator_predictions/logistic_loss/zeros_like"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss/Neg"
  op: "Neg"
  input: "discriminator_predictions/Squeeze"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss/Select_1"
  op: "Select"
  input: "discriminator_predictions/logistic_loss/GreaterEqual"
  input: "discriminator_predictions/logistic_loss/Neg"
  input: "discriminator_predictions/Squeeze"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss/mul"
  op: "Mul"
  input: "discriminator_predictions/Squeeze"
  input: "discriminator_predictions/Cast_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss/sub"
  op: "Sub"
  input: "discriminator_predictions/logistic_loss/Select"
  input: "discriminator_predictions/logistic_loss/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss/Exp"
  op: "Exp"
  input: "discriminator_predictions/logistic_loss/Select_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss/Log1p"
  op: "Log1p"
  input: "discriminator_predictions/logistic_loss/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/logistic_loss"
  op: "AddV2"
  input: "discriminator_predictions/logistic_loss/sub"
  input: "discriminator_predictions/logistic_loss/Log1p"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/mul"
  op: "Mul"
  input: "discriminator_predictions/logistic_loss"
  input: "discriminator_predictions/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "discriminator_predictions/Sum"
  op: "Sum"
  input: "discriminator_predictions/mul"
  input: "discriminator_predictions/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "discriminator_predictions/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "discriminator_predictions/Sum_1"
  op: "Sum"
  input: "discriminator_predictions/Cast"
  input: "discriminator_predictions/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "discriminator_predictions/add/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "discriminator_predictions/add"
  op: "AddV2"
  input: "discriminator_predictions/add/x"
  input: "discriminator_predictions/Sum_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/truediv"
  op: "RealDiv"
  input: "discriminator_predictions/Sum"
  input: "discriminator_predictions/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "discriminator_predictions/Sum_2"
  op: "Sum"
  input: "discriminator_predictions/mul"
  input: "discriminator_predictions/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "discriminator_predictions/Const_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "discriminator_predictions/Sum_3"
  op: "Sum"
  input: "discriminator_predictions/Cast"
  input: "discriminator_predictions/Const_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "discriminator_predictions/add_1/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "discriminator_predictions/add_1"
  op: "AddV2"
  input: "discriminator_predictions/add_1/x"
  input: "discriminator_predictions/Sum_3"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/truediv_1"
  op: "RealDiv"
  input: "discriminator_predictions/Sum_2"
  input: "discriminator_predictions/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/Sigmoid"
  op: "Sigmoid"
  input: "discriminator_predictions/Squeeze"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/Sign"
  op: "Sign"
  input: "discriminator_predictions/Squeeze"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/add_2/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/add_2"
  op: "AddV2"
  input: "discriminator_predictions/Sign"
  input: "discriminator_predictions/add_2/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/truediv_2/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/truediv_2"
  op: "RealDiv"
  input: "discriminator_predictions/add_2"
  input: "discriminator_predictions/truediv_2/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/Round"
  op: "Round"
  input: "discriminator_predictions/truediv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "discriminator_predictions/Cast_2"
  op: "Cast"
  input: "discriminator_predictions/Round"
  attr {
    key: "DstT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_20/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 50.0
      }
    }
  }
}
node {
  name: "mul_20"
  op: "Mul"
  input: "mul_20/x"
  input: "discriminator_predictions/truediv_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_11"
  op: "AddV2"
  input: "mul_19"
  input: "mul_20"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ArgMax_2/dimension"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "ArgMax_2"
  op: "ArgMax"
  input: "StopGradient_2"
  input: "ArgMax_2/dimension"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "output_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "PolynomialDecay/initial_learning_rate"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.00019999999494757503
      }
    }
  }
}
node {
  name: "PolynomialDecay/Cast/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "PolynomialDecay/Cast_1/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "PolynomialDecay/Cast_2/ReadVariableOp"
  op: "ReadVariableOp"
  input: "PolynomialDecay/Cast_2/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "PolynomialDecay/Cast_2/ReadVariableOp/Enter"
  op: "Enter"
  input: "global_step"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "PolynomialDecay/Cast_2"
  op: "Cast"
  input: "PolynomialDecay/Cast_2/ReadVariableOp"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "PolynomialDecay/Cast_3/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1000000
      }
    }
  }
}
node {
  name: "PolynomialDecay/Cast_3"
  op: "Cast"
  input: "PolynomialDecay/Cast_3/x"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "PolynomialDecay/Minimum"
  op: "Minimum"
  input: "PolynomialDecay/Cast_2"
  input: "PolynomialDecay/Cast_3"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "PolynomialDecay/truediv"
  op: "RealDiv"
  input: "PolynomialDecay/Minimum"
  input: "PolynomialDecay/Cast_3"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "PolynomialDecay/sub"
  op: "Sub"
  input: "PolynomialDecay/initial_learning_rate"
  input: "PolynomialDecay/Cast/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "PolynomialDecay/sub_1/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "PolynomialDecay/sub_1"
  op: "Sub"
  input: "PolynomialDecay/sub_1/x"
  input: "PolynomialDecay/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "PolynomialDecay/Pow"
  op: "Pow"
  input: "PolynomialDecay/sub_1"
  input: "PolynomialDecay/Cast_1/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "PolynomialDecay/Mul"
  op: "Mul"
  input: "PolynomialDecay/sub"
  input: "PolynomialDecay/Pow"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "PolynomialDecay"
  op: "AddV2"
  input: "PolynomialDecay/Mul"
  input: "PolynomialDecay/Cast/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Cast_9/ReadVariableOp"
  op: "ReadVariableOp"
  input: "PolynomialDecay/Cast_2/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "Cast_9"
  op: "Cast"
  input: "Cast_9/ReadVariableOp"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Cast_10/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 20000
      }
    }
  }
}
node {
  name: "Cast_10"
  op: "Cast"
  input: "Cast_10/x"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_2"
  op: "RealDiv"
  input: "Cast_9"
  input: "Cast_10"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Minimum_7/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "Minimum_7"
  op: "Minimum"
  input: "Minimum_7/x"
  input: "truediv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_21"
  op: "Mul"
  input: "PolynomialDecay"
  input: "Minimum_7"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/grad_ys_0/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "gradients/grad_ys_0"
  op: "Fill"
  input: "gradients/Shape"
  input: "gradients/grad_ys_0/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "gradients/mul_19_grad/Mul"
  op: "Mul"
  input: "gradients/grad_ys_0"
  input: "generator_predictions/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/mul_19_grad/Mul_1"
  op: "Mul"
  input: "gradients/grad_ys_0"
  input: "mul_19/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/mul_20_grad/Mul"
  op: "Mul"
  input: "gradients/grad_ys_0"
  input: "discriminator_predictions/truediv_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/mul_20_grad/Mul_1"
  op: "Mul"
  input: "gradients/grad_ys_0"
  input: "mul_20/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator_predictions/truediv_grad/Shape"
  input: "gradients/generator_predictions/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/mul_19_grad/Mul_1"
  input: "generator_predictions/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator_predictions/truediv_grad/RealDiv"
  input: "gradients/generator_predictions/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/truediv_grad/Sum"
  input: "gradients/generator_predictions/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/Neg"
  op: "Neg"
  input: "generator_predictions/Sum_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator_predictions/truediv_grad/Neg"
  input: "generator_predictions/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator_predictions/truediv_grad/RealDiv_1"
  input: "generator_predictions/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/mul"
  op: "Mul"
  input: "gradients/mul_19_grad/Mul_1"
  input: "gradients/generator_predictions/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator_predictions/truediv_grad/mul"
  input: "gradients/generator_predictions/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator_predictions/truediv_grad/Sum_1"
  input: "gradients/generator_predictions/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/discriminator_predictions/truediv_1_grad/Shape"
  input: "gradients/discriminator_predictions/truediv_1_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/mul_20_grad/Mul_1"
  input: "discriminator_predictions/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/Sum"
  op: "Sum"
  input: "gradients/discriminator_predictions/truediv_1_grad/RealDiv"
  input: "gradients/discriminator_predictions/truediv_1_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/discriminator_predictions/truediv_1_grad/Sum"
  input: "gradients/discriminator_predictions/truediv_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/Neg"
  op: "Neg"
  input: "discriminator_predictions/Sum_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/discriminator_predictions/truediv_1_grad/Neg"
  input: "discriminator_predictions/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/discriminator_predictions/truediv_1_grad/RealDiv_1"
  input: "discriminator_predictions/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/mul"
  op: "Mul"
  input: "gradients/mul_20_grad/Mul_1"
  input: "gradients/discriminator_predictions/truediv_1_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/Sum_1"
  op: "Sum"
  input: "gradients/discriminator_predictions/truediv_1_grad/mul"
  input: "gradients/discriminator_predictions/truediv_1_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/discriminator_predictions/truediv_1_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/discriminator_predictions/truediv_1_grad/Sum_1"
  input: "gradients/discriminator_predictions/truediv_1_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_1_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/truediv_grad/Reshape"
  input: "gradients/generator_predictions/Sum_1_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_1_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000O\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_1_grad/Tile"
  op: "Tile"
  input: "gradients/generator_predictions/Sum_1_grad/Reshape"
  input: "gradients/generator_predictions/Sum_1_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/Sum_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/Sum_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/discriminator_predictions/truediv_1_grad/Reshape"
  input: "gradients/discriminator_predictions/Sum_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/Sum_2_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/Sum_2_grad/Tile"
  op: "Tile"
  input: "gradients/discriminator_predictions/Sum_2_grad/Reshape"
  input: "gradients/discriminator_predictions/Sum_2_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator_predictions/Sum_1_grad/Tile"
  input: "generator_predictions/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/mul_2_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator_predictions/Sum_1_grad/Tile"
  input: "Cast_3"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/mul_grad/Mul"
  op: "Mul"
  input: "gradients/discriminator_predictions/Sum_2_grad/Tile"
  input: "discriminator_predictions/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/mul_grad/Mul_1"
  op: "Mul"
  input: "gradients/discriminator_predictions/Sum_2_grad/Tile"
  input: "discriminator_predictions/logistic_loss"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/Neg_grad/Neg"
  op: "Neg"
  input: "gradients/generator_predictions/mul_2_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_grad/Maximum/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_grad/Maximum/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_grad/Maximum"
  op: "Maximum"
  input: "gradients/generator_predictions/Sum_grad/Maximum/x"
  input: "gradients/generator_predictions/Sum_grad/Maximum/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_grad/floordiv/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000Y\304\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_grad/floordiv"
  op: "FloorDiv"
  input: "gradients/generator_predictions/Sum_grad/floordiv/x"
  input: "gradients/generator_predictions/Sum_grad/Maximum"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/Neg_grad/Neg"
  input: "gradients/generator_predictions/Sum_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\001\000\000\000Y\304\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Sum_grad/Tile"
  op: "Tile"
  input: "gradients/generator_predictions/Sum_grad/Reshape"
  input: "gradients/generator_predictions/Sum_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/sub_grad/Neg"
  op: "Neg"
  input: "gradients/discriminator_predictions/mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Log1p_grad/add/x"
  op: "Const"
  input: "^gradients/discriminator_predictions/mul_grad/Mul"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Log1p_grad/add"
  op: "AddV2"
  input: "gradients/discriminator_predictions/logistic_loss/Log1p_grad/add/x"
  input: "discriminator_predictions/logistic_loss/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Log1p_grad/Reciprocal"
  op: "Reciprocal"
  input: "gradients/discriminator_predictions/logistic_loss/Log1p_grad/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Log1p_grad/mul"
  op: "Mul"
  input: "gradients/discriminator_predictions/mul_grad/Mul"
  input: "gradients/discriminator_predictions/logistic_loss/Log1p_grad/Reciprocal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator_predictions/Sum_grad/Tile"
  input: "generator_predictions/one_hot"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator_predictions/Sum_grad/Tile"
  input: "generator_predictions/LogSoftmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Select_grad/zeros_like/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Select_grad/zeros_like/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Select_grad/zeros_like"
  op: "Fill"
  input: "gradients/discriminator_predictions/logistic_loss/Select_grad/zeros_like/shape_as_tensor"
  input: "gradients/discriminator_predictions/logistic_loss/Select_grad/zeros_like/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Select_grad/Select"
  op: "Select"
  input: "discriminator_predictions/logistic_loss/GreaterEqual"
  input: "gradients/discriminator_predictions/mul_grad/Mul"
  input: "gradients/discriminator_predictions/logistic_loss/Select_grad/zeros_like"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Select_grad/Select_1"
  op: "Select"
  input: "discriminator_predictions/logistic_loss/GreaterEqual"
  input: "gradients/discriminator_predictions/logistic_loss/Select_grad/zeros_like"
  input: "gradients/discriminator_predictions/mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/mul_grad/Mul"
  op: "Mul"
  input: "gradients/discriminator_predictions/logistic_loss/sub_grad/Neg"
  input: "discriminator_predictions/Cast_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/mul_grad/Mul_1"
  op: "Mul"
  input: "gradients/discriminator_predictions/logistic_loss/sub_grad/Neg"
  input: "discriminator_predictions/Squeeze"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Exp_grad/mul"
  op: "Mul"
  input: "gradients/discriminator_predictions/logistic_loss/Log1p_grad/mul"
  input: "discriminator_predictions/logistic_loss/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LogSoftmax_grad/Exp"
  op: "Exp"
  input: "generator_predictions/LogSoftmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LogSoftmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LogSoftmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator_predictions/mul_1_grad/Mul"
  input: "gradients/generator_predictions/LogSoftmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator_predictions/LogSoftmax_grad/mul"
  op: "Mul"
  input: "gradients/generator_predictions/LogSoftmax_grad/Sum"
  input: "gradients/generator_predictions/LogSoftmax_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LogSoftmax_grad/sub"
  op: "Sub"
  input: "gradients/generator_predictions/mul_1_grad/Mul"
  input: "gradients/generator_predictions/LogSoftmax_grad/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Select_1_grad/zeros_like/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: " \000\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Select_1_grad/zeros_like/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Select_1_grad/zeros_like"
  op: "Fill"
  input: "gradients/discriminator_predictions/logistic_loss/Select_1_grad/zeros_like/shape_as_tensor"
  input: "gradients/discriminator_predictions/logistic_loss/Select_1_grad/zeros_like/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Select_1_grad/Select"
  op: "Select"
  input: "discriminator_predictions/logistic_loss/GreaterEqual"
  input: "gradients/discriminator_predictions/logistic_loss/Exp_grad/mul"
  input: "gradients/discriminator_predictions/logistic_loss/Select_1_grad/zeros_like"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Select_1_grad/Select_1"
  op: "Select"
  input: "discriminator_predictions/logistic_loss/GreaterEqual"
  input: "gradients/discriminator_predictions/logistic_loss/Select_1_grad/zeros_like"
  input: "gradients/discriminator_predictions/logistic_loss/Exp_grad/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator_predictions/LogSoftmax_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/logistic_loss/Neg_grad/Neg"
  op: "Neg"
  input: "gradients/discriminator_predictions/logistic_loss/Select_1_grad/Select"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator_predictions/LogSoftmax_grad/sub"
  input: "generator_predictions/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator_predictions/LogSoftmax_grad/sub"
  input: "generator_predictions/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "Y\304\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/strided_slice/stack"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/strided_slice/stack_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -2
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/strided_slice/stack_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/strided_slice"
  op: "StridedSlice"
  input: "gradients/generator_predictions/MatMul_grad/Shape"
  input: "gradients/generator_predictions/MatMul_grad/strided_slice/stack"
  input: "gradients/generator_predictions/MatMul_grad/strided_slice/stack_1"
  input: "gradients/generator_predictions/MatMul_grad/strided_slice/stack_2"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 1
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 0
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/strided_slice_1/stack"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/strided_slice_1/stack_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -2
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/strided_slice_1/stack_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/strided_slice_1"
  op: "StridedSlice"
  input: "gradients/generator_predictions/MatMul_grad/Shape_1"
  input: "gradients/generator_predictions/MatMul_grad/strided_slice_1/stack"
  input: "gradients/generator_predictions/MatMul_grad/strided_slice_1/stack_1"
  input: "gradients/generator_predictions/MatMul_grad/strided_slice_1/stack_2"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 1
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 0
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator_predictions/MatMul_grad/strided_slice"
  input: "gradients/generator_predictions/MatMul_grad/strided_slice_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/Sum"
  op: "Sum"
  input: "gradients/generator_predictions/MatMul_grad/MatMul"
  input: "gradients/generator_predictions/MatMul_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/MatMul_grad/Sum"
  input: "gradients/generator_predictions/MatMul_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator_predictions/MatMul_grad/MatMul_1"
  input: "gradients/generator_predictions/MatMul_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/MatMul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator_predictions/MatMul_grad/Sum_1"
  input: "gradients/generator_predictions/MatMul_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN"
  op: "AddN"
  input: "gradients/discriminator_predictions/logistic_loss/Select_grad/Select"
  input: "gradients/discriminator_predictions/logistic_loss/mul_grad/Mul"
  input: "gradients/discriminator_predictions/logistic_loss/Select_1_grad/Select_1"
  input: "gradients/discriminator_predictions/logistic_loss/Neg_grad/Neg"
  attr {
    key: "N"
    value {
      i: 4
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/logistic_loss/Select_grad/Select"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/Squeeze_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/Squeeze_grad/Reshape"
  op: "Reshape"
  input: "gradients/AddN"
  input: "gradients/discriminator_predictions/Squeeze_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense_1/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/discriminator_predictions/Squeeze_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator_predictions/MatMul_grad/Reshape"
  input: "generator_predictions/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator_predictions/MatMul_grad/Reshape"
  input: "generator_predictions/dense/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/generator_predictions/MatMul_grad/Reshape"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/generator_predictions/MatMul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense_1/Tensordot_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense_1/Tensordot_grad/Reshape"
  op: "Reshape"
  input: "gradients/discriminator_predictions/Squeeze_grad/Reshape"
  input: "gradients/discriminator_predictions/dense_1/Tensordot_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator_predictions/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator_predictions/LayerNorm/moments/mean"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense_1/Tensordot/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/discriminator_predictions/dense_1/Tensordot_grad/Reshape"
  input: "discriminator_predictions/dense_1/Tensordot/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense_1/Tensordot/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "discriminator_predictions/dense_1/Tensordot/Reshape"
  input: "gradients/discriminator_predictions/dense_1/Tensordot_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_1"
  op: "AddN"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_1"
  input: "generator_predictions/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator_predictions/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense_1/Tensordot/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense_1/Tensordot/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/discriminator_predictions/dense_1/Tensordot/MatMul_grad/MatMul"
  input: "gradients/discriminator_predictions/dense_1/Tensordot/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator_predictions/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/discriminator_predictions/dense_1/Tensordot/Reshape_grad/Reshape"
  input: "discriminator_predictions/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/discriminator_predictions/dense_1/Tensordot/Reshape_grad/Reshape"
  input: "discriminator_predictions/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_grad/BroadcastGradientArgs/s0_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/discriminator_predictions/dense/mul_grad/BroadcastGradientArgs/s0_1"
  input: "gradients/discriminator_predictions/dense/mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/discriminator_predictions/dense/mul_1_grad/Mul_1"
  input: "discriminator_predictions/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/discriminator_predictions/dense/mul_grad/Mul"
  input: "gradients/discriminator_predictions/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/discriminator_predictions/dense/mul_grad/Sum"
  input: "gradients/discriminator_predictions/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "discriminator_predictions/dense/mul/x"
  input: "gradients/discriminator_predictions/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Maximum/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Maximum/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Maximum"
  op: "Maximum"
  input: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Maximum/x"
  input: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Maximum/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/variance_grad/floordiv/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/variance_grad/floordiv"
  op: "FloorDiv"
  input: "gradients/generator_predictions/LayerNorm/moments/variance_grad/floordiv/x"
  input: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Maximum"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator_predictions/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/discriminator_predictions/dense/mul_grad/Mul_1"
  input: "gradients/discriminator_predictions/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/discriminator_predictions/dense/add_grad/Sum"
  input: "gradients/discriminator_predictions/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator_predictions/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator_predictions/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator_predictions/dense/mul_1"
  input: "generator_predictions/LayerNorm/moments/StopGradient"
  input: "^gradients/generator_predictions/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/discriminator_predictions/dense/mul_grad/Mul_1"
  input: "gradients/discriminator_predictions/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Erf_grad/Square"
  op: "Square"
  input: "discriminator_predictions/dense/truediv"
  input: "^gradients/discriminator_predictions/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/discriminator_predictions/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/discriminator_predictions/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/discriminator_predictions/dense/Erf_grad/mul"
  input: "gradients/discriminator_predictions/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/discriminator_predictions/dense/truediv_grad/Shape"
  input: "gradients/discriminator_predictions/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/discriminator_predictions/dense/Erf_grad/mul_1"
  input: "discriminator_predictions/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/discriminator_predictions/dense/truediv_grad/RealDiv"
  input: "gradients/discriminator_predictions/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/discriminator_predictions/dense/truediv_grad/Sum"
  input: "gradients/discriminator_predictions/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/Neg"
  op: "Neg"
  input: "discriminator_predictions/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/discriminator_predictions/dense/truediv_grad/Neg"
  input: "discriminator_predictions/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/discriminator_predictions/dense/truediv_grad/RealDiv_1"
  input: "discriminator_predictions/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/discriminator_predictions/dense/Erf_grad/mul_1"
  input: "gradients/discriminator_predictions/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/discriminator_predictions/dense/truediv_grad/mul"
  input: "gradients/discriminator_predictions/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/discriminator_predictions/dense/truediv_grad/Sum_1"
  input: "gradients/discriminator_predictions/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator_predictions/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator_predictions/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator_predictions/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator_predictions/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator_predictions/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_2"
  op: "AddN"
  input: "gradients/discriminator_predictions/dense/mul_1_grad/Mul"
  input: "gradients/discriminator_predictions/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_3"
  op: "AddN"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator_predictions/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator_predictions/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_3"
  input: "generator_predictions/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_3"
  input: "generator_predictions/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Tensordot_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Tensordot_grad/Reshape"
  op: "Reshape"
  input: "gradients/AddN_2"
  input: "gradients/discriminator_predictions/dense/Tensordot_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_grad/BroadcastGradientArgs/s0_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator_predictions/dense/mul_grad/BroadcastGradientArgs/s0_1"
  input: "gradients/generator_predictions/dense/mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator_predictions/dense/mul_1_grad/Mul_1"
  input: "generator_predictions/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator_predictions/dense/mul_grad/Mul"
  input: "gradients/generator_predictions/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/dense/mul_grad/Sum"
  input: "gradients/generator_predictions/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator_predictions/dense/mul/x"
  input: "gradients/generator_predictions/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Tensordot/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/discriminator_predictions/dense/Tensordot_grad/Reshape"
  input: "discriminator_predictions/dense/Tensordot/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "discriminator_predictions/dense/Tensordot/Reshape"
  input: "gradients/discriminator_predictions/dense/Tensordot_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator_predictions/dense/mul_grad/Mul_1"
  input: "gradients/generator_predictions/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/dense/add_grad/Sum"
  input: "gradients/generator_predictions/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Tensordot/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/discriminator_predictions/dense/Tensordot/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/discriminator_predictions/dense/Tensordot/MatMul_grad/MatMul"
  input: "gradients/discriminator_predictions/dense/Tensordot/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator_predictions/dense/mul_grad/Mul_1"
  input: "gradients/generator_predictions/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Erf_grad/Square"
  op: "Square"
  input: "generator_predictions/dense/truediv"
  input: "^gradients/generator_predictions/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator_predictions/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator_predictions/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator_predictions/dense/Erf_grad/mul"
  input: "gradients/generator_predictions/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/strided_slice_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\014\000\000\000 \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/strided_slice_grad/StridedSliceGrad/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\014\000\000\000 \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/strided_slice_grad/StridedSliceGrad/begin"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/strided_slice_grad/StridedSliceGrad/end"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/strided_slice_grad/StridedSliceGrad/strides"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/strided_slice_grad/StridedSliceGrad"
  op: "StridedSliceGrad"
  input: "gradients/electra/encoder/strided_slice_grad/StridedSliceGrad/shape"
  input: "gradients/electra/encoder/strided_slice_grad/StridedSliceGrad/begin"
  input: "gradients/electra/encoder/strided_slice_grad/StridedSliceGrad/end"
  input: "gradients/electra/encoder/strided_slice_grad/StridedSliceGrad/strides"
  input: "gradients/discriminator_predictions/dense/Tensordot/Reshape_grad/Reshape"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 12
          }
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 1
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator_predictions/dense/truediv_grad/Shape"
  input: "gradients/generator_predictions/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator_predictions/dense/Erf_grad/mul_1"
  input: "generator_predictions/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator_predictions/dense/truediv_grad/RealDiv"
  input: "gradients/generator_predictions/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/dense/truediv_grad/Sum"
  input: "gradients/generator_predictions/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator_predictions/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator_predictions/dense/truediv_grad/Neg"
  input: "generator_predictions/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator_predictions/dense/truediv_grad/RealDiv_1"
  input: "generator_predictions/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator_predictions/dense/Erf_grad/mul_1"
  input: "gradients/generator_predictions/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator_predictions/dense/truediv_grad/mul"
  input: "gradients/generator_predictions/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator_predictions/dense/truediv_grad/Sum_1"
  input: "gradients/generator_predictions/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/stack_1_grad/unstack"
  op: "Unpack"
  input: "gradients/electra/encoder/strided_slice_grad/StridedSliceGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
  attr {
    key: "num"
    value {
      i: 12
    }
  }
}
node {
  name: "gradients/AddN_4"
  op: "AddN"
  input: "gradients/generator_predictions/dense/mul_1_grad/Mul"
  input: "gradients/generator_predictions/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_4"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack"
  input: "gradients/electra/encoder/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack:1"
  input: "gradients/electra/encoder/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_4_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_4_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack:2"
  input: "gradients/electra/encoder/Reshape_4_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_5_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_5_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack:3"
  input: "gradients/electra/encoder/Reshape_5_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_6_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_6_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack:4"
  input: "gradients/electra/encoder/Reshape_6_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_7_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_7_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack:5"
  input: "gradients/electra/encoder/Reshape_7_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_8_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_8_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack:6"
  input: "gradients/electra/encoder/Reshape_8_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_9_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_9_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack:7"
  input: "gradients/electra/encoder/Reshape_9_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_10_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_10_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack:8"
  input: "gradients/electra/encoder/Reshape_10_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_11_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_11_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack:9"
  input: "gradients/electra/encoder/Reshape_11_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_12_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_12_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack:10"
  input: "gradients/electra/encoder/Reshape_12_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_13_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_13_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/stack_1_grad/unstack:11"
  input: "gradients/electra/encoder/Reshape_13_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Tensordot_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\340\t\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Tensordot_grad/Reshape"
  op: "Reshape"
  input: "gradients/AddN_4"
  input: "gradients/generator_predictions/dense/Tensordot_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Tensordot/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator_predictions/dense/Tensordot_grad/Reshape"
  input: "generator_predictions/dense/Tensordot/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator_predictions/dense/Tensordot/Reshape"
  input: "gradients/generator_predictions/dense/Tensordot_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/Reshape_13_grad/Reshape"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/Reshape_13_grad/Reshape"
  input: "electra/encoder/layer_11/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/Reshape_13_grad/Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/Reshape_13_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Tensordot/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000O\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/dense/Tensordot/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/dense/Tensordot/MatMul_grad/MatMul"
  input: "gradients/generator_predictions/dense/Tensordot/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 79
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\340\t\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/dense/Tensordot/Reshape_grad/Reshape"
  input: "gradients/generator_predictions/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_5"
  op: "AddN"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_5"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_5"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\000\000\000\000\001\000\000\000\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/Cast"
  op: "Cast"
  input: "gradients/generator_predictions/GatherV2_grad/Shape"
  attr {
    key: "DstT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/Size"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 2528
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/ExpandDims/dim"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/ExpandDims"
  op: "ExpandDims"
  input: "gradients/generator_predictions/GatherV2_grad/Size"
  input: "gradients/generator_predictions/GatherV2_grad/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/strided_slice/stack"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/strided_slice/stack_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/strided_slice/stack_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/strided_slice"
  op: "StridedSlice"
  input: "gradients/generator_predictions/GatherV2_grad/Cast"
  input: "gradients/generator_predictions/GatherV2_grad/strided_slice/stack"
  input: "gradients/generator_predictions/GatherV2_grad/strided_slice/stack_1"
  input: "gradients/generator_predictions/GatherV2_grad/strided_slice/stack_2"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 1
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 0
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/concat/axis"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/concat"
  op: "ConcatV2"
  input: "gradients/generator_predictions/GatherV2_grad/ExpandDims"
  input: "gradients/generator_predictions/GatherV2_grad/strided_slice"
  input: "gradients/generator_predictions/GatherV2_grad/concat/axis"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/Reshape_2_grad/Reshape"
  input: "gradients/generator_predictions/GatherV2_grad/concat"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/GatherV2_grad/Reshape_1"
  op: "Reshape"
  input: "generator_predictions/Reshape"
  input: "gradients/generator_predictions/GatherV2_grad/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2528
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/Reshape_1_grad/strided_slice/stack"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Reshape_1_grad/strided_slice/stack_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Reshape_1_grad/strided_slice/stack_2"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Reshape_1_grad/strided_slice"
  op: "StridedSlice"
  input: "gradients/generator_predictions/GatherV2_grad/Cast"
  input: "gradients/generator_predictions/Reshape_1_grad/strided_slice/stack"
  input: "gradients/generator_predictions/Reshape_1_grad/strided_slice/stack_1"
  input: "gradients/generator_predictions/Reshape_1_grad/strided_slice/stack_2"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 1
    }
  }
}
node {
  name: "gradients/generator_predictions/Reshape_1_grad/UnsortedSegmentSum"
  op: "UnsortedSegmentSum"
  input: "gradients/generator_predictions/GatherV2_grad/Reshape"
  input: "gradients/generator_predictions/GatherV2_grad/Reshape_1"
  input: "gradients/generator_predictions/Reshape_1_grad/strided_slice"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tnumsegments"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator_predictions/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator_predictions/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator_predictions/Reshape_1_grad/UnsortedSegmentSum"
  input: "gradients/generator_predictions/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/strided_slice_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\014\000\000\000 \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/strided_slice_grad/StridedSliceGrad/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\014\000\000\000 \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/strided_slice_grad/StridedSliceGrad/begin"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/strided_slice_grad/StridedSliceGrad/end"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/strided_slice_grad/StridedSliceGrad/strides"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/strided_slice_grad/StridedSliceGrad"
  op: "StridedSliceGrad"
  input: "gradients/generator/encoder/strided_slice_grad/StridedSliceGrad/shape"
  input: "gradients/generator/encoder/strided_slice_grad/StridedSliceGrad/begin"
  input: "gradients/generator/encoder/strided_slice_grad/StridedSliceGrad/end"
  input: "gradients/generator/encoder/strided_slice_grad/StridedSliceGrad/strides"
  input: "gradients/generator_predictions/Reshape_1_grad/Reshape"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 12
          }
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 1
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum"
  op: "Maximum"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum/x"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/floordiv/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/floordiv"
  op: "FloorDiv"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/floordiv/x"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/stack_1_grad/unstack"
  op: "Unpack"
  input: "gradients/generator/encoder/strided_slice_grad/StridedSliceGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
  attr {
    key: "num"
    value {
      i: 12
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_11/output/add"
  input: "electra/encoder/layer_11/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack"
  input: "gradients/generator/encoder/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack:1"
  input: "gradients/generator/encoder/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_4_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_4_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack:2"
  input: "gradients/generator/encoder/Reshape_4_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_5_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_5_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack:3"
  input: "gradients/generator/encoder/Reshape_5_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_6_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_6_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack:4"
  input: "gradients/generator/encoder/Reshape_6_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_7_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_7_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack:5"
  input: "gradients/generator/encoder/Reshape_7_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_8_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_8_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack:6"
  input: "gradients/generator/encoder/Reshape_8_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_9_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_9_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack:7"
  input: "gradients/generator/encoder/Reshape_9_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_10_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_10_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack:8"
  input: "gradients/generator/encoder/Reshape_10_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_11_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_11_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack:9"
  input: "gradients/generator/encoder/Reshape_11_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_12_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_12_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack:10"
  input: "gradients/generator/encoder/Reshape_12_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_13_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_13_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/stack_1_grad/unstack:11"
  input: "gradients/generator/encoder/Reshape_13_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/Reshape_13_grad/Reshape"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/Reshape_13_grad/Reshape"
  input: "generator/encoder/layer_11/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/Reshape_13_grad/Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/Reshape_13_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_6"
  op: "AddN"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_6"
  input: "electra/encoder/layer_11/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_6"
  input: "electra/encoder/layer_11/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_7"
  op: "AddN"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_7"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_7"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_11/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_11/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_11/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_11/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_11/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum"
  op: "Maximum"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum/x"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/floordiv/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/floordiv"
  op: "FloorDiv"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/floordiv/x"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Maximum"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_11/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_11/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_11/output/add"
  input: "generator/encoder/layer_11/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_11/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs/s0_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs/s0_1"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_11/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_11/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_11/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_8"
  op: "AddN"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_11/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_11/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_11/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_11/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_8"
  input: "generator/encoder/layer_11/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_8"
  input: "generator/encoder/layer_11/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_9"
  op: "AddN"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_9"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_11/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_11/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_9"
  input: "electra/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_9"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_10"
  op: "AddN"
  input: "gradients/AddN_6"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_11/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_11/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_11/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_10"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_10"
  input: "electra/encoder/layer_11/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_10"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_10"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_11/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_11/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs/s0_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs/s0_1"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_11/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_11"
  op: "AddN"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_11"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_11"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_11/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_11/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_11/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_11/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_11/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_11/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_12"
  op: "AddN"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_12"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_11/attention/output/add"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_12"
  input: "generator/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_12"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_13"
  op: "AddN"
  input: "gradients/AddN_8"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_13"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_13"
  input: "generator/encoder/layer_11/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_13"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_13"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_14"
  op: "AddN"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_14"
  input: "electra/encoder/layer_11/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_14"
  input: "electra/encoder/layer_11/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_15"
  op: "AddN"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_15"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_15"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_11/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_11/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_11/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_11/attention/output/add"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_11/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_11/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_11/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_11/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_11/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_11/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_11/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_11/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_11/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_11/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_11/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_16"
  op: "AddN"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_11/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_16"
  input: "generator/encoder/layer_11/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_16"
  input: "generator/encoder/layer_11/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_11/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_11/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_11/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_11/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_11/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_11/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_11/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_11/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_11/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_11/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_11/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_11/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_11/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_11/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_11/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_11/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_11/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_11/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_11/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_11/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_11/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_11/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_11/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_11/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_11/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_11/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_11/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_11/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_11/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_11/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_11/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_11/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_11/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_11/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_17"
  op: "AddN"
  input: "gradients/electra/encoder/Reshape_12_grad/Reshape"
  input: "gradients/AddN_14"
  input: "gradients/electra/encoder/layer_11/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_11/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_11/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/Reshape_12_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_11/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_11/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_11/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_17"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_17"
  input: "electra/encoder/layer_10/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_17"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_17"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\004\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s0"
  input: "gradients/generator/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_11/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_11/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_11/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_11/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_11/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_18"
  op: "AddN"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_18"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_18"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_11/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_11/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_11/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_11/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_11/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_11/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_11/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_11/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_11/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_11/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_10/output/add"
  input: "electra/encoder/layer_10/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_11/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_19"
  op: "AddN"
  input: "gradients/generator/encoder/Reshape_12_grad/Reshape"
  input: "gradients/AddN_16"
  input: "gradients/generator/encoder/layer_11/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_11/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_11/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/Reshape_12_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_19"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_19"
  input: "generator/encoder/layer_10/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_19"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_19"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_20"
  op: "AddN"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_20"
  input: "electra/encoder/layer_10/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_20"
  input: "electra/encoder/layer_10/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_21"
  op: "AddN"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_21"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_21"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_10/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_10/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_10/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_10/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_10/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_10/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_10/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_10/output/add"
  input: "generator/encoder/layer_10/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_10/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_10/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_10/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_10/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_22"
  op: "AddN"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_10/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_10/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_10/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_10/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_22"
  input: "generator/encoder/layer_10/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_22"
  input: "generator/encoder/layer_10/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_23"
  op: "AddN"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_10/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_10/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_23"
  input: "electra/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_24"
  op: "AddN"
  input: "gradients/AddN_20"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_10/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_10/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_10/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_24"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_24"
  input: "electra/encoder/layer_10/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_24"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_24"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_10/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_10/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_10/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_25"
  op: "AddN"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_25"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_25"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_10/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_10/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_10/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_10/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_10/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_10/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_26"
  op: "AddN"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_26"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_10/attention/output/add"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_26"
  input: "generator/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_26"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_27"
  op: "AddN"
  input: "gradients/AddN_22"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_27"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_27"
  input: "generator/encoder/layer_10/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_27"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_27"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_28"
  op: "AddN"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_28"
  input: "electra/encoder/layer_10/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_28"
  input: "electra/encoder/layer_10/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_29"
  op: "AddN"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_29"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_29"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_10/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_10/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_10/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_10/attention/output/add"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_10/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_10/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_10/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_10/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_10/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_10/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_10/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_10/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_10/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_10/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_10/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_30"
  op: "AddN"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_10/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_30"
  input: "generator/encoder/layer_10/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_30"
  input: "generator/encoder/layer_10/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_10/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_10/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_10/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_10/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_10/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_10/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_10/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_10/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_10/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_10/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_10/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_10/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_10/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_10/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_10/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_10/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_10/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_10/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_10/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_10/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_10/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_10/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_10/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_10/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_10/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_10/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_10/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_10/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_10/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_10/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_10/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_10/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_10/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_10/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_31"
  op: "AddN"
  input: "gradients/electra/encoder/Reshape_11_grad/Reshape"
  input: "gradients/AddN_28"
  input: "gradients/electra/encoder/layer_10/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_10/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_10/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/Reshape_11_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_10/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_10/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_10/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_31"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_31"
  input: "electra/encoder/layer_9/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_31"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_31"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_10/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_10/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_10/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_10/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_10/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_32"
  op: "AddN"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_32"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_32"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_10/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_10/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_10/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_10/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_10/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_10/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_10/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_10/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_10/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_10/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_9/output/add"
  input: "electra/encoder/layer_9/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_10/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_33"
  op: "AddN"
  input: "gradients/generator/encoder/Reshape_11_grad/Reshape"
  input: "gradients/AddN_30"
  input: "gradients/generator/encoder/layer_10/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_10/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_10/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/Reshape_11_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_33"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_33"
  input: "generator/encoder/layer_9/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_33"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_33"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_34"
  op: "AddN"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_34"
  input: "electra/encoder/layer_9/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_34"
  input: "electra/encoder/layer_9/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_35"
  op: "AddN"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_35"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_35"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_9/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_9/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_9/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_9/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_9/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_9/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_9/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_9/output/add"
  input: "generator/encoder/layer_9/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_9/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_9/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_9/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_9/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_36"
  op: "AddN"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_9/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_9/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_9/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_9/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_36"
  input: "generator/encoder/layer_9/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_36"
  input: "generator/encoder/layer_9/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_37"
  op: "AddN"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_37"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_9/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_9/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_37"
  input: "electra/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_37"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_38"
  op: "AddN"
  input: "gradients/AddN_34"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_9/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_9/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_9/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_38"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_38"
  input: "electra/encoder/layer_9/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_38"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_38"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_9/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_9/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_9/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_39"
  op: "AddN"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_39"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_39"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_9/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_9/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_9/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_9/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_9/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_9/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_40"
  op: "AddN"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_40"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_9/attention/output/add"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_40"
  input: "generator/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_40"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_41"
  op: "AddN"
  input: "gradients/AddN_36"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_41"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_41"
  input: "generator/encoder/layer_9/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_41"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_41"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_42"
  op: "AddN"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_42"
  input: "electra/encoder/layer_9/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_42"
  input: "electra/encoder/layer_9/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_43"
  op: "AddN"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_43"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_43"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_9/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_9/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_9/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_9/attention/output/add"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_9/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_9/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_9/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_9/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_9/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_9/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_9/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_9/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_9/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_9/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_9/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_44"
  op: "AddN"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_9/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_44"
  input: "generator/encoder/layer_9/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_44"
  input: "generator/encoder/layer_9/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_9/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_9/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_9/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_9/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_9/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_9/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_9/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_9/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_9/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_9/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_9/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_9/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_9/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_9/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_9/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_9/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_9/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_9/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_9/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_9/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_9/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_9/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_9/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_9/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_9/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_9/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_9/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_9/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_9/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_9/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_9/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_9/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_9/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_9/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_45"
  op: "AddN"
  input: "gradients/electra/encoder/Reshape_10_grad/Reshape"
  input: "gradients/AddN_42"
  input: "gradients/electra/encoder/layer_9/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_9/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_9/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/Reshape_10_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_9/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_9/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_9/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_45"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_45"
  input: "electra/encoder/layer_8/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_45"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_45"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_9/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_9/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_9/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_9/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_9/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_46"
  op: "AddN"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_46"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_46"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_9/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_9/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_9/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_9/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_9/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_9/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_9/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_9/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_9/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_9/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_8/output/add"
  input: "electra/encoder/layer_8/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_9/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_47"
  op: "AddN"
  input: "gradients/generator/encoder/Reshape_10_grad/Reshape"
  input: "gradients/AddN_44"
  input: "gradients/generator/encoder/layer_9/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_9/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_9/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/Reshape_10_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_47"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_47"
  input: "generator/encoder/layer_8/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_47"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_47"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_48"
  op: "AddN"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_48"
  input: "electra/encoder/layer_8/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_48"
  input: "electra/encoder/layer_8/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_49"
  op: "AddN"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_49"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_49"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_8/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_8/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_8/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_8/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_8/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_8/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_8/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_8/output/add"
  input: "generator/encoder/layer_8/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_8/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_8/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_8/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_8/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_50"
  op: "AddN"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_8/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_8/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_8/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_8/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_50"
  input: "generator/encoder/layer_8/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_50"
  input: "generator/encoder/layer_8/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_51"
  op: "AddN"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_51"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_8/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_8/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_51"
  input: "electra/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_51"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_52"
  op: "AddN"
  input: "gradients/AddN_48"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_8/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_8/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_8/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_52"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_52"
  input: "electra/encoder/layer_8/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_52"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_52"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_8/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_8/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_8/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_53"
  op: "AddN"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_53"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_53"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_8/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_8/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_8/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_8/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_8/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_8/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_54"
  op: "AddN"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_54"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_8/attention/output/add"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_54"
  input: "generator/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_54"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_55"
  op: "AddN"
  input: "gradients/AddN_50"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_55"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_55"
  input: "generator/encoder/layer_8/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_55"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_55"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_56"
  op: "AddN"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_56"
  input: "electra/encoder/layer_8/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_56"
  input: "electra/encoder/layer_8/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_57"
  op: "AddN"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_57"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_57"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_8/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_8/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_8/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_8/attention/output/add"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_8/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_8/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_8/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_8/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_8/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_8/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_8/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_8/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_8/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_8/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_8/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_58"
  op: "AddN"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_8/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_58"
  input: "generator/encoder/layer_8/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_58"
  input: "generator/encoder/layer_8/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_8/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_8/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_8/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_8/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_8/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_8/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_8/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_8/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_8/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_8/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_8/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_8/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_8/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_8/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_8/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_8/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_8/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_8/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_8/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_8/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_8/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_8/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_8/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_8/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_8/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_8/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_8/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_8/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_8/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_8/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_8/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_8/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_8/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_8/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_59"
  op: "AddN"
  input: "gradients/electra/encoder/Reshape_9_grad/Reshape"
  input: "gradients/AddN_56"
  input: "gradients/electra/encoder/layer_8/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_8/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_8/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/Reshape_9_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_8/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_8/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_8/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_59"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_59"
  input: "electra/encoder/layer_7/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_59"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_59"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_8/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_8/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_8/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_8/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_8/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_60"
  op: "AddN"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_60"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_60"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_8/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_8/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_8/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_8/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_8/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_8/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_8/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_8/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_8/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_8/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_7/output/add"
  input: "electra/encoder/layer_7/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_8/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_61"
  op: "AddN"
  input: "gradients/generator/encoder/Reshape_9_grad/Reshape"
  input: "gradients/AddN_58"
  input: "gradients/generator/encoder/layer_8/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_8/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_8/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/Reshape_9_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_61"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_61"
  input: "generator/encoder/layer_7/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_61"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_61"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_62"
  op: "AddN"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_62"
  input: "electra/encoder/layer_7/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_62"
  input: "electra/encoder/layer_7/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_63"
  op: "AddN"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_63"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_63"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_7/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_7/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_7/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_7/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_7/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_7/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_7/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_7/output/add"
  input: "generator/encoder/layer_7/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_7/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_7/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_7/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_7/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_64"
  op: "AddN"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_7/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_7/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_7/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_7/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_64"
  input: "generator/encoder/layer_7/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_64"
  input: "generator/encoder/layer_7/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_65"
  op: "AddN"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_65"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_7/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_7/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_65"
  input: "electra/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_65"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_66"
  op: "AddN"
  input: "gradients/AddN_62"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_7/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_7/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_7/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_66"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_66"
  input: "electra/encoder/layer_7/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_66"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_66"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_7/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_7/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_7/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_67"
  op: "AddN"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_67"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_67"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_7/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_7/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_7/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_7/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_7/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_7/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_68"
  op: "AddN"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_68"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_7/attention/output/add"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_68"
  input: "generator/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_68"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_69"
  op: "AddN"
  input: "gradients/AddN_64"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_69"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_69"
  input: "generator/encoder/layer_7/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_69"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_69"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_70"
  op: "AddN"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_70"
  input: "electra/encoder/layer_7/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_70"
  input: "electra/encoder/layer_7/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_71"
  op: "AddN"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_71"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_71"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_7/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_7/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_7/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_7/attention/output/add"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_7/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_7/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_7/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_7/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_7/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_7/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_7/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_7/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_7/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_7/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_7/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_72"
  op: "AddN"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_7/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_72"
  input: "generator/encoder/layer_7/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_72"
  input: "generator/encoder/layer_7/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_7/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_7/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_7/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_7/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_7/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_7/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_7/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_7/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_7/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_7/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_7/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_7/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_7/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_7/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_7/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_7/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_7/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_7/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_7/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_7/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_7/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_7/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_7/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_7/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_7/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_7/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_7/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_7/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_7/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_7/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_7/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_7/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_7/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_7/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_73"
  op: "AddN"
  input: "gradients/electra/encoder/Reshape_8_grad/Reshape"
  input: "gradients/AddN_70"
  input: "gradients/electra/encoder/layer_7/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_7/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_7/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/Reshape_8_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_7/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_7/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_7/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_73"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_73"
  input: "electra/encoder/layer_6/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_73"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_73"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_7/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_7/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_7/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_7/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_7/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_74"
  op: "AddN"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_74"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_74"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_7/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_7/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_7/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_7/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_7/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_7/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_7/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_7/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_7/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_7/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_6/output/add"
  input: "electra/encoder/layer_6/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_7/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_75"
  op: "AddN"
  input: "gradients/generator/encoder/Reshape_8_grad/Reshape"
  input: "gradients/AddN_72"
  input: "gradients/generator/encoder/layer_7/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_7/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_7/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/Reshape_8_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_75"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_75"
  input: "generator/encoder/layer_6/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_75"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_75"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_76"
  op: "AddN"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_76"
  input: "electra/encoder/layer_6/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_76"
  input: "electra/encoder/layer_6/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_77"
  op: "AddN"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_77"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_77"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_6/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_6/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_6/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_6/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_6/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_6/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_6/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_6/output/add"
  input: "generator/encoder/layer_6/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_6/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_6/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_6/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_6/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_78"
  op: "AddN"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_6/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_6/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_6/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_6/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_78"
  input: "generator/encoder/layer_6/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_78"
  input: "generator/encoder/layer_6/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_79"
  op: "AddN"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_79"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_6/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_6/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_79"
  input: "electra/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_79"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_80"
  op: "AddN"
  input: "gradients/AddN_76"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_6/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_6/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_6/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_80"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_80"
  input: "electra/encoder/layer_6/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_80"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_80"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_6/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_6/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_6/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_81"
  op: "AddN"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_81"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_81"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_6/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_6/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_6/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_6/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_6/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_6/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_82"
  op: "AddN"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_82"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_6/attention/output/add"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_82"
  input: "generator/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_82"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_83"
  op: "AddN"
  input: "gradients/AddN_78"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_83"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_83"
  input: "generator/encoder/layer_6/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_83"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_83"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_84"
  op: "AddN"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_84"
  input: "electra/encoder/layer_6/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_84"
  input: "electra/encoder/layer_6/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_85"
  op: "AddN"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_85"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_85"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_6/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_6/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_6/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_6/attention/output/add"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_6/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_6/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_6/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_6/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_6/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_6/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_6/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_6/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_6/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_6/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_6/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_86"
  op: "AddN"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_6/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_86"
  input: "generator/encoder/layer_6/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_86"
  input: "generator/encoder/layer_6/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_6/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_6/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_6/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_6/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_6/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_6/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_6/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_6/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_6/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_6/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_6/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_6/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_6/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_6/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_6/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_6/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_6/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_6/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_6/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_6/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_6/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_6/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_6/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_6/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_6/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_6/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_6/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_6/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_6/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_6/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_6/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_6/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_6/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_6/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_87"
  op: "AddN"
  input: "gradients/electra/encoder/Reshape_7_grad/Reshape"
  input: "gradients/AddN_84"
  input: "gradients/electra/encoder/layer_6/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_6/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_6/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/Reshape_7_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_6/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_6/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_6/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_87"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_87"
  input: "electra/encoder/layer_5/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_87"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_87"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_6/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_6/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_6/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_6/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_6/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_88"
  op: "AddN"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_88"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_88"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_6/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_6/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_6/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_6/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_6/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_6/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_6/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_6/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_6/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_6/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_5/output/add"
  input: "electra/encoder/layer_5/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_6/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_89"
  op: "AddN"
  input: "gradients/generator/encoder/Reshape_7_grad/Reshape"
  input: "gradients/AddN_86"
  input: "gradients/generator/encoder/layer_6/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_6/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_6/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/Reshape_7_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_89"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_89"
  input: "generator/encoder/layer_5/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_89"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_89"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_90"
  op: "AddN"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_90"
  input: "electra/encoder/layer_5/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_90"
  input: "electra/encoder/layer_5/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_91"
  op: "AddN"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_91"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_91"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_5/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_5/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_5/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_5/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_5/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_5/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_5/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_5/output/add"
  input: "generator/encoder/layer_5/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_5/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_5/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_5/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_5/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_92"
  op: "AddN"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_5/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_5/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_5/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_5/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_92"
  input: "generator/encoder/layer_5/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_92"
  input: "generator/encoder/layer_5/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_93"
  op: "AddN"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_93"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_5/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_5/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_93"
  input: "electra/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_93"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_94"
  op: "AddN"
  input: "gradients/AddN_90"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_5/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_5/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_5/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_94"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_94"
  input: "electra/encoder/layer_5/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_94"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_94"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_5/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_5/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_5/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_95"
  op: "AddN"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_95"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_95"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_5/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_5/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_5/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_5/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_5/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_5/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_96"
  op: "AddN"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_96"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_5/attention/output/add"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_96"
  input: "generator/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_96"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_97"
  op: "AddN"
  input: "gradients/AddN_92"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_97"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_97"
  input: "generator/encoder/layer_5/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_97"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_97"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_98"
  op: "AddN"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_98"
  input: "electra/encoder/layer_5/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_98"
  input: "electra/encoder/layer_5/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_99"
  op: "AddN"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_99"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_99"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_5/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_5/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_5/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_5/attention/output/add"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_5/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_5/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_5/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_5/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_5/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_5/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_5/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_5/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_5/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_5/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_5/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_100"
  op: "AddN"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_5/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_100"
  input: "generator/encoder/layer_5/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_100"
  input: "generator/encoder/layer_5/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_5/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_5/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_5/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_5/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_5/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_5/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_5/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_5/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_5/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_5/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_5/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_5/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_5/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_5/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_5/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_5/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_5/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_5/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_5/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_5/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_5/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_5/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_5/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_5/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_5/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_5/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_5/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_5/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_5/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_5/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_5/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_5/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_5/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_5/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_101"
  op: "AddN"
  input: "gradients/electra/encoder/Reshape_6_grad/Reshape"
  input: "gradients/AddN_98"
  input: "gradients/electra/encoder/layer_5/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_5/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_5/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/Reshape_6_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_5/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_5/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_5/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_101"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_101"
  input: "electra/encoder/layer_4/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_101"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_5/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_5/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_5/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_5/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_5/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_102"
  op: "AddN"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_102"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_5/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_5/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_5/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_5/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_5/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_5/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_5/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_5/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_5/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_5/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_4/output/add"
  input: "electra/encoder/layer_4/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_5/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_103"
  op: "AddN"
  input: "gradients/generator/encoder/Reshape_6_grad/Reshape"
  input: "gradients/AddN_100"
  input: "gradients/generator/encoder/layer_5/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_5/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_5/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/Reshape_6_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_103"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_103"
  input: "generator/encoder/layer_4/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_103"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_104"
  op: "AddN"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_104"
  input: "electra/encoder/layer_4/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_104"
  input: "electra/encoder/layer_4/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_105"
  op: "AddN"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_105"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_105"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_4/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_4/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_4/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_4/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_4/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_4/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_4/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_4/output/add"
  input: "generator/encoder/layer_4/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_4/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_4/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_4/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_4/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_106"
  op: "AddN"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_4/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_4/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_4/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_4/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_106"
  input: "generator/encoder/layer_4/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_106"
  input: "generator/encoder/layer_4/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_107"
  op: "AddN"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_4/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_4/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_107"
  input: "electra/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_108"
  op: "AddN"
  input: "gradients/AddN_104"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_4/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_4/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_4/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_108"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_108"
  input: "electra/encoder/layer_4/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_108"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_4/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_4/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_4/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_109"
  op: "AddN"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_109"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_4/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_4/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_4/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_4/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_4/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_4/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_110"
  op: "AddN"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_110"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_4/attention/output/add"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_110"
  input: "generator/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_110"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_111"
  op: "AddN"
  input: "gradients/AddN_106"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_111"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_111"
  input: "generator/encoder/layer_4/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_111"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_112"
  op: "AddN"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_112"
  input: "electra/encoder/layer_4/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_112"
  input: "electra/encoder/layer_4/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_113"
  op: "AddN"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_113"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_4/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_4/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_4/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_4/attention/output/add"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_4/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_4/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_4/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_4/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_4/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_4/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_4/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_4/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_4/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_4/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_4/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_114"
  op: "AddN"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_4/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_114"
  input: "generator/encoder/layer_4/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_114"
  input: "generator/encoder/layer_4/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_4/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_4/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_4/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_4/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_4/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_4/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_4/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_4/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_4/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_4/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_4/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_4/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_4/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_4/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_4/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_4/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_4/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_4/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_4/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_4/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_4/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_4/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_4/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_4/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_4/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_4/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_4/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_4/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_4/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_4/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_4/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_4/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_4/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_4/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_115"
  op: "AddN"
  input: "gradients/electra/encoder/Reshape_5_grad/Reshape"
  input: "gradients/AddN_112"
  input: "gradients/electra/encoder/layer_4/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_4/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_4/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/Reshape_5_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_4/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_4/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_4/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_115"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_115"
  input: "electra/encoder/layer_3/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_115"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_4/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_4/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_4/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_4/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_4/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_116"
  op: "AddN"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_116"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_116"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_4/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_4/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_4/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_4/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_4/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_4/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_4/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_4/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_4/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_4/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_3/output/add"
  input: "electra/encoder/layer_3/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_4/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_117"
  op: "AddN"
  input: "gradients/generator/encoder/Reshape_5_grad/Reshape"
  input: "gradients/AddN_114"
  input: "gradients/generator/encoder/layer_4/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_4/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_4/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/Reshape_5_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_117"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_117"
  input: "generator/encoder/layer_3/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_117"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_118"
  op: "AddN"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_118"
  input: "electra/encoder/layer_3/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_118"
  input: "electra/encoder/layer_3/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_119"
  op: "AddN"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_119"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_119"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_3/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_3/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_3/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_3/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_3/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_3/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_3/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_3/output/add"
  input: "generator/encoder/layer_3/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_3/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_3/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_3/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_3/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_120"
  op: "AddN"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_3/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_3/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_3/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_3/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_120"
  input: "generator/encoder/layer_3/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_120"
  input: "generator/encoder/layer_3/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_121"
  op: "AddN"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_3/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_3/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_121"
  input: "electra/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_122"
  op: "AddN"
  input: "gradients/AddN_118"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_3/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_3/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_3/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_122"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_122"
  input: "electra/encoder/layer_3/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_122"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_3/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_3/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_3/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_123"
  op: "AddN"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_123"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_3/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_3/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_3/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_3/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_3/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_3/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_124"
  op: "AddN"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_3/attention/output/add"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_124"
  input: "generator/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_125"
  op: "AddN"
  input: "gradients/AddN_120"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_125"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_125"
  input: "generator/encoder/layer_3/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_125"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_126"
  op: "AddN"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_126"
  input: "electra/encoder/layer_3/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_126"
  input: "electra/encoder/layer_3/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_127"
  op: "AddN"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_127"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_3/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_3/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_3/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_3/attention/output/add"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_3/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_3/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_3/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_3/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_3/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_3/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_3/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_3/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_3/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_3/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_3/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_128"
  op: "AddN"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_3/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_128"
  input: "generator/encoder/layer_3/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_128"
  input: "generator/encoder/layer_3/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_3/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_3/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_3/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_3/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_3/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_3/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_3/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_3/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_3/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_3/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_3/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_3/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_3/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_3/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_3/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_3/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_3/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_3/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_3/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_3/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_3/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_3/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_3/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_3/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_3/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_3/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_3/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_3/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_3/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_3/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_3/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_3/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_3/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_3/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_129"
  op: "AddN"
  input: "gradients/electra/encoder/Reshape_4_grad/Reshape"
  input: "gradients/AddN_126"
  input: "gradients/electra/encoder/layer_3/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_3/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_3/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/Reshape_4_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_3/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_3/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_3/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_129"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_129"
  input: "electra/encoder/layer_2/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_129"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_3/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_3/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_3/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_3/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_3/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_130"
  op: "AddN"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_130"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_3/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_3/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_3/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_3/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_3/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_3/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_3/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_3/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_3/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_3/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_2/output/add"
  input: "electra/encoder/layer_2/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_3/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_131"
  op: "AddN"
  input: "gradients/generator/encoder/Reshape_4_grad/Reshape"
  input: "gradients/AddN_128"
  input: "gradients/generator/encoder/layer_3/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_3/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_3/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/Reshape_4_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_131"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_131"
  input: "generator/encoder/layer_2/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_131"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_132"
  op: "AddN"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_132"
  input: "electra/encoder/layer_2/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_132"
  input: "electra/encoder/layer_2/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_133"
  op: "AddN"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_133"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_2/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_2/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_2/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_2/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_2/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_2/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_2/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_2/output/add"
  input: "generator/encoder/layer_2/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_2/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_2/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_2/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_2/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_134"
  op: "AddN"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_2/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_2/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_2/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_2/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_134"
  input: "generator/encoder/layer_2/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_134"
  input: "generator/encoder/layer_2/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_135"
  op: "AddN"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_2/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_2/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_135"
  input: "electra/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_136"
  op: "AddN"
  input: "gradients/AddN_132"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_2/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_2/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_2/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_136"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_136"
  input: "electra/encoder/layer_2/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_136"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_2/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_2/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_2/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_137"
  op: "AddN"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_137"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_137"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_2/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_2/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_2/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_2/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_2/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_2/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_138"
  op: "AddN"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_2/attention/output/add"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_138"
  input: "generator/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_139"
  op: "AddN"
  input: "gradients/AddN_134"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_139"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_139"
  input: "generator/encoder/layer_2/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_139"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_139"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_140"
  op: "AddN"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_140"
  input: "electra/encoder/layer_2/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_140"
  input: "electra/encoder/layer_2/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_141"
  op: "AddN"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_141"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_2/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_2/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_2/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_2/attention/output/add"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_2/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_2/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_2/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_2/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_2/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_2/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_2/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_2/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_2/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_2/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_2/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_142"
  op: "AddN"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_2/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_142"
  input: "generator/encoder/layer_2/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_142"
  input: "generator/encoder/layer_2/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_2/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_2/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_2/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_2/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_2/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_2/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_2/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_2/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_2/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_2/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_2/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_2/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_2/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_2/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_2/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_2/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_2/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_2/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_2/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_2/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_2/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_2/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_2/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_2/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_2/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_2/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_2/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_2/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_2/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_2/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_2/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_2/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_2/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_2/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_143"
  op: "AddN"
  input: "gradients/electra/encoder/Reshape_3_grad/Reshape"
  input: "gradients/AddN_140"
  input: "gradients/electra/encoder/layer_2/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_2/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_2/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/Reshape_3_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_2/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_2/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_2/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_143"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_143"
  input: "electra/encoder/layer_1/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_143"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_2/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_2/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_2/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_2/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_2/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_144"
  op: "AddN"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_144"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_2/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_2/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_2/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_2/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_2/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_2/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_2/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_2/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_2/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_2/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_1/output/add"
  input: "electra/encoder/layer_1/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_2/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_145"
  op: "AddN"
  input: "gradients/generator/encoder/Reshape_3_grad/Reshape"
  input: "gradients/AddN_142"
  input: "gradients/generator/encoder/layer_2/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_2/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_2/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/Reshape_3_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_145"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_145"
  input: "generator/encoder/layer_1/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_145"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_146"
  op: "AddN"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_146"
  input: "electra/encoder/layer_1/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_146"
  input: "electra/encoder/layer_1/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_147"
  op: "AddN"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_147"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_1/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_1/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_1/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_1/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_1/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_1/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_1/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_1/output/add"
  input: "generator/encoder/layer_1/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_1/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_1/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_1/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_1/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_148"
  op: "AddN"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_1/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_1/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_1/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_1/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_148"
  input: "generator/encoder/layer_1/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_148"
  input: "generator/encoder/layer_1/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_149"
  op: "AddN"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_1/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_1/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_149"
  input: "electra/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_150"
  op: "AddN"
  input: "gradients/AddN_146"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_1/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_1/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_1/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_150"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_150"
  input: "electra/encoder/layer_1/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_150"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_1/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_1/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_1/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_151"
  op: "AddN"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_151"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_1/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_1/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_1/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_1/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_1/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_1/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_152"
  op: "AddN"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_1/attention/output/add"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_152"
  input: "generator/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_153"
  op: "AddN"
  input: "gradients/AddN_148"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_153"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_153"
  input: "generator/encoder/layer_1/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_153"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_153"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_154"
  op: "AddN"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_154"
  input: "electra/encoder/layer_1/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_154"
  input: "electra/encoder/layer_1/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_155"
  op: "AddN"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_155"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_1/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_1/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_1/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_1/attention/output/add"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_1/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_1/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_1/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_1/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_1/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_1/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_1/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_1/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_1/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_1/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_1/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_156"
  op: "AddN"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_1/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_156"
  input: "generator/encoder/layer_1/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_156"
  input: "generator/encoder/layer_1/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_1/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_1/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_1/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_1/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_1/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_1/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_1/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_1/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_1/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_1/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_1/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_1/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_1/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_1/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_1/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_1/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_1/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_1/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_1/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_1/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_1/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_1/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_1/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_1/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_1/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_1/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_1/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_1/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_1/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_1/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_1/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_1/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/encoder/layer_1/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_1/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_157"
  op: "AddN"
  input: "gradients/electra/encoder/Reshape_2_grad/Reshape"
  input: "gradients/AddN_154"
  input: "gradients/electra/encoder/layer_1/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_1/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_1/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/Reshape_2_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_1/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_1/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_1/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_157"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_157"
  input: "electra/encoder/layer_0/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_157"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_1/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_1/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_1/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_1/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_1/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_158"
  op: "AddN"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_158"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_158"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_1/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_1/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_1/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_1/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_1/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_1/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_1/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_1/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_1/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_1/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_0/output/add"
  input: "electra/encoder/layer_0/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "gradients/generator/encoder/layer_1/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_159"
  op: "AddN"
  input: "gradients/generator/encoder/Reshape_2_grad/Reshape"
  input: "gradients/AddN_156"
  input: "gradients/generator/encoder/layer_1/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_1/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_1/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 5
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/Reshape_2_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_159"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_159"
  input: "generator/encoder/layer_0/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_159"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_160"
  op: "AddN"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_160"
  input: "electra/encoder/layer_0/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_160"
  input: "electra/encoder/layer_0/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_161"
  op: "AddN"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_161"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_0/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_0/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_0/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_0/intermediate/dense/mul_1"
  input: "gradients/electra/encoder/layer_0/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_0/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/output/dense/MatMul_grad/MatMul"
  input: "electra/encoder/layer_0/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_0/output/add"
  input: "generator/encoder/layer_0/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_0/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1"
  input: "electra/encoder/layer_0/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/intermediate/dense/mul/x"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/add_grad/Sum"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "electra/encoder/layer_0/intermediate/dense/truediv"
  input: "^gradients/electra/encoder/layer_0/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/mul"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_162"
  op: "AddN"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Shape"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/mul_1"
  input: "electra/encoder/layer_0/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Sum"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "electra/encoder/layer_0/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Neg"
  input: "electra/encoder/layer_0/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_1"
  input: "electra/encoder/layer_0/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/mul"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_162"
  input: "generator/encoder/layer_0/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_162"
  input: "generator/encoder/layer_0/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_163"
  op: "AddN"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_0/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_0/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_163"
  input: "electra/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/AddN_164"
  op: "AddN"
  input: "gradients/AddN_160"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_0/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_0/intermediate/dense/mul_1"
  input: "gradients/generator/encoder/layer_0/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_164"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_164"
  input: "electra/encoder/layer_0/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_164"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_164"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_0/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/output/dense/MatMul_grad/MatMul"
  input: "generator/encoder/layer_0/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/moments/mean"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1"
  input: "generator/encoder/layer_0/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/intermediate/dense/mul/x"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_165"
  op: "AddN"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_165"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/add_grad/Sum"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.128379225730896
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/Square"
  op: "Square"
  input: "generator/encoder/layer_0/intermediate/dense/truediv"
  input: "^gradients/generator/encoder/layer_0/intermediate/dense/mul_grad/Mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/Exp"
  op: "Exp"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/mul"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/Exp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Shape"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/mul_1"
  input: "generator/encoder/layer_0/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/BroadcastGradientArgs"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Sum"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Neg"
  op: "Neg"
  input: "generator/encoder/layer_0/intermediate/dense/BiasAdd"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_1"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Neg"
  input: "generator/encoder/layer_0/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_2"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_1"
  input: "generator/encoder/layer_0/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/Erf_grad/mul_1"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/RealDiv_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/mul"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/BroadcastGradientArgs:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Sum_1"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_166"
  op: "AddN"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/truediv_grad/Reshape"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/intermediate/dense/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/AddN_166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/encoder/layer_0/attention/output/add"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/AddN_166"
  input: "generator/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1"
  input: "gradients/AddN_166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_167"
  op: "AddN"
  input: "gradients/AddN_162"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_167"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_167"
  input: "generator/encoder/layer_0/attention/output/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_167"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/AddN_167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_168"
  op: "AddN"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/moments/mean"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_168"
  input: "electra/encoder/layer_0/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_168"
  input: "electra/encoder/layer_0/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_169"
  op: "AddN"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_169"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 256
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_0/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/output/dense/BiasAdd"
  input: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/layer_0/attention/self/Reshape_3"
  input: "gradients/electra/encoder/layer_0/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "generator/encoder/layer_0/attention/output/add"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/moments/StopGradient"
  input: "^gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_0/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/electra/encoder/layer_0/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_0/attention/self/transpose_3_grad/transpose"
  input: "electra/encoder/layer_0/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "electra/encoder/layer_0/attention/self/dropout/Mul_1"
  input: "gradients/electra/encoder/layer_0/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\001\000\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 256.0
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_0/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/MatMul_1_grad/MatMul"
  input: "electra/encoder/layer_0/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_0/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_0/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_0/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_170"
  op: "AddN"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_1_grad/Mul"
  input: "electra/encoder/layer_0/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/self/Softmax"
  input: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/self/transpose_2_grad/transpose"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_170"
  input: "generator/encoder/layer_0/attention/output/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/AddN_170"
  input: "generator/encoder/layer_0/attention/output/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Mul"
  input: "electra/encoder/layer_0/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/mul"
  input: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/electra/encoder/layer_0/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/sub"
  input: "electra/encoder/layer_0/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_0/attention/output/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/output/dense/BiasAdd"
  input: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/mul_1"
  input: "gradients/electra/encoder/layer_0/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/self/add_grad/Sum"
  input: "gradients/electra/encoder/layer_0/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_2_grad/Reshape"
  input: "electra/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/Reshape_1"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/mul_1"
  input: "electra/encoder/layer_0/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/encoder/layer_0/attention/self/MatMul"
  input: "gradients/electra/encoder/layer_0/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Mul_1"
  input: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Sum"
  input: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/layer_0/attention/self/Reshape_3"
  input: "gradients/generator/encoder/layer_0/attention/output/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_0/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/electra/encoder/layer_0/attention/self/Mul_grad/Mul"
  input: "electra/encoder/layer_0/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Reshape_3_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\004\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Reshape_3_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_3_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_0/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_0/attention/self/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_0/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "electra/encoder/layer_0/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/electra/encoder/layer_0/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/electra/encoder/layer_0/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/transpose_3_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_0/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/transpose_3_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_3_grad/Reshape"
  input: "gradients/generator/encoder/layer_0/attention/self/transpose_3_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/self/transpose_grad/transpose"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/encoder/layer_0/attention/self/transpose_1_grad/transpose"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/MatMul_1_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_0/attention/self/transpose_3_grad/transpose"
  input: "generator/encoder/layer_0/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/MatMul_1_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "generator/encoder/layer_0/attention/self/dropout/Mul_1"
  input: "gradients/generator/encoder/layer_0/attention/self/transpose_3_grad/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_0/attention/self/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/MatMul_1_grad/MatMul"
  input: "generator/encoder/layer_0/attention/self/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/transpose_2_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_0/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/transpose_2_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_0/attention/self/MatMul_1_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_0/attention/self/transpose_2_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_grad/Reshape"
  input: "electra/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/Reshape_1"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_1_grad/Reshape"
  input: "electra/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/encoder/Reshape_1"
  input: "gradients/electra/encoder/layer_0/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_1_grad/Mul"
  input: "generator/encoder/layer_0/attention/self/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/self/Softmax"
  input: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/self/transpose_2_grad/transpose"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_171"
  op: "AddN"
  input: "gradients/AddN_168"
  input: "gradients/electra/encoder/layer_0/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_0/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/electra/encoder/layer_0/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 4
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/encoder/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/AddN_171"
  input: "gradients/electra/encoder/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Mul"
  input: "generator/encoder/layer_0/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/mul"
  input: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/sub"
  op: "Sub"
  input: "gradients/generator/encoder/layer_0/attention/self/dropout/Mul_grad/Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/Sum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/mul_1"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/sub"
  input: "generator/encoder/layer_0/attention/self/Softmax"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/encoder/Reshape_1_grad/Reshape"
  input: "electra/embeddings_3/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/encoder/Reshape_1_grad/Reshape"
  input: "electra/embeddings_3/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/add_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/mul_1"
  input: "gradients/generator/encoder/layer_0/attention/self/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\000\002\000\000\000\002\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/self/add_grad/Sum"
  input: "gradients/generator/encoder/layer_0/attention/self/add_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/value/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_2_grad/Reshape"
  input: "generator/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/Reshape_1"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_2_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/embeddings_3/dropout/Mul_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/embeddings_3/dropout/Mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/embeddings_3/dropout/Mul_1_grad/Mul"
  input: "electra/embeddings_3/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/embeddings_3/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/embeddings_3/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_3/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/embeddings_3/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_3/dropout/Mul_grad/Sum"
  input: "gradients/electra/embeddings_3/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/mul_1"
  input: "generator/encoder/layer_0/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Mul_1"
  op: "Mul"
  input: "generator/encoder/layer_0/attention/self/MatMul"
  input: "gradients/generator/encoder/layer_0/attention/self/Softmax_grad/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Mul_1"
  input: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Sum"
  input: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/MatMul_grad/MatMul"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_0/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/MatMul_grad/MatMul_1"
  op: "BatchMatMulV2"
  input: "gradients/generator/encoder/layer_0/attention/self/Mul_grad/Mul"
  input: "generator/encoder/layer_0/attention/self/transpose"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 4
          }
          dim {
            size: 512
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "adj_x"
    value {
      b: true
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/embeddings_3/dropout/Mul_grad/Mul"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/embeddings_3/dropout/Mul_grad/Mul"
  input: "electra/embeddings_3/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_3/dropout/Mul_grad/Mul"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/electra/embeddings_3/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/transpose_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_0/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/transpose_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_0/attention/self/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_0/attention/self/transpose_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/transpose_1_grad/InvertPermutation"
  op: "InvertPermutation"
  input: "generator/encoder/layer_0/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/transpose_1_grad/transpose"
  op: "Transpose"
  input: "gradients/generator/encoder/layer_0/attention/self/MatMul_grad/MatMul_1"
  input: "gradients/generator/encoder/layer_0/attention/self/transpose_1_grad/InvertPermutation"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 4
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/embeddings_3/LayerNorm/moments/mean"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/self/transpose_grad/transpose"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/layer_0/attention/self/transpose_1_grad/transpose"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_172"
  op: "AddN"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_172"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/embeddings_3/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/embeddings_3/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/query/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_grad/Reshape"
  input: "generator/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/Reshape_1"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/key/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_1_grad/Reshape"
  input: "generator/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/encoder/Reshape_1"
  input: "gradients/generator/encoder/layer_0/attention/self/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_173"
  op: "AddN"
  input: "gradients/AddN_170"
  input: "gradients/generator/encoder/layer_0/attention/self/value/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_0/attention/self/query/MatMul_grad/MatMul"
  input: "gradients/generator/encoder/layer_0/attention/self/key/MatMul_grad/MatMul"
  attr {
    key: "N"
    value {
      i: 4
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/encoder/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/AddN_173"
  input: "gradients/generator/encoder/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Maximum/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Maximum/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Maximum"
  op: "Maximum"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Maximum/x"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Maximum/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/floordiv/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/floordiv"
  op: "FloorDiv"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/floordiv/x"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Maximum"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/embeddings_project/BiasAdd_grad/BiasAddGrad"
  op: "BiasAddGrad"
  input: "gradients/generator/encoder/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/embeddings_3/add_1"
  input: "electra/embeddings_3/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/embeddings_3/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/embeddings_project/Tensordot_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/embeddings_project/Tensordot_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/encoder/Reshape_1_grad/Reshape"
  input: "gradients/generator/embeddings_project/Tensordot_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/embeddings_project/Tensordot/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/generator/embeddings_project/Tensordot_grad/Reshape"
  input: "generator/embeddings_project/Tensordot/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/generator/embeddings_project/Tensordot/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "generator/embeddings_project/Tensordot/Reshape"
  input: "gradients/generator/embeddings_project/Tensordot_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/generator/embeddings_project/Tensordot/Reshape_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/generator/embeddings_project/Tensordot/Reshape_grad/Reshape"
  op: "Reshape"
  input: "gradients/generator/embeddings_project/Tensordot/MatMul_grad/MatMul"
  input: "gradients/generator/embeddings_project/Tensordot/Reshape_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_174"
  op: "AddN"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/embeddings_3/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/add_1_grad/BroadcastGradientArgs/s0"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/add_1_grad/BroadcastGradientArgs/s1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/add_1_grad/BroadcastGradientArgs"
  op: "BroadcastGradientArgs"
  input: "gradients/electra/embeddings_3/add_1_grad/BroadcastGradientArgs/s0"
  input: "gradients/electra/embeddings_3/add_1_grad/BroadcastGradientArgs/s1"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/add_1_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/add_1_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_174"
  input: "gradients/electra/embeddings_3/add_1_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/add_1_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/add_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_3/add_1_grad/Sum"
  input: "gradients/electra/embeddings_3/add_1_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/dropout/Mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/generator/embeddings_project/Tensordot/Reshape_grad/Reshape"
  input: "electra/embeddings_1/dropout/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/dropout/Mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/generator/embeddings_project/Tensordot/Reshape_grad/Reshape"
  input: "electra/embeddings_1/dropout/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_3/add_1_grad/Reshape"
  input: "gradients/electra/embeddings_3/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/dropout/Mul_grad/Mul"
  op: "Mul"
  input: "gradients/electra/embeddings_1/dropout/Mul_1_grad/Mul"
  input: "electra/embeddings_1/dropout/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/dropout/Mul_grad/Mul_1"
  op: "Mul"
  input: "electra/embeddings_1/LayerNorm/batchnorm/add_1"
  input: "gradients/electra/embeddings_1/dropout/Mul_1_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/dropout/Mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/dropout/Mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_1/dropout/Mul_grad/Mul_1"
  input: "gradients/electra/embeddings_1/dropout/Mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/dropout/Mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/dropout/Mul_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/dropout/Mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_1/dropout/Mul_grad/Sum"
  input: "gradients/electra/embeddings_1/dropout/Mul_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_2/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_2/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/AddN_174"
  input: "gradients/electra/embeddings_2/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/AddN_174"
  input: "gradients/electra/embeddings_3/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/Slice_grad/Rank"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/Slice_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/Slice_grad/zeros_like/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/Slice_grad/zeros_like/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/Slice_grad/zeros_like"
  op: "Fill"
  input: "gradients/electra/embeddings_3/Slice_grad/zeros_like/shape_as_tensor"
  input: "gradients/electra/embeddings_3/Slice_grad/zeros_like/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/Slice_grad/XlaDynamicUpdateSlice"
  op: "XlaDynamicUpdateSlice"
  input: "gradients/electra/embeddings_3/Slice_grad/zeros_like"
  input: "gradients/electra/embeddings_3/Reshape_2_grad/Reshape"
  input: "electra/embeddings_3/Slice/begin"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_2/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/embeddings_2/Reshape_1_grad/Reshape"
  input: "electra/embeddings_2/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/embeddings_2/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/embeddings_2/one_hot"
  input: "gradients/electra/embeddings_2/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/embeddings_3/Reshape_1_grad/Reshape"
  input: "electra/embeddings_3/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/embeddings_3/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/embeddings_3/one_hot"
  input: "gradients/electra/embeddings_3/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_1_grad/Mul"
  op: "Mul"
  input: "gradients/electra/embeddings_1/dropout/Mul_grad/Mul"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  op: "Mul"
  input: "gradients/electra/embeddings_1/dropout/Mul_grad/Mul"
  input: "electra/embeddings_1/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_1/dropout/Mul_grad/Mul"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Sum"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Neg"
  op: "Neg"
  input: "gradients/electra/embeddings_1/dropout/Mul_grad/Mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Mul"
  op: "Mul"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Neg"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Mul"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Sum"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  op: "Mul"
  input: "electra/embeddings_1/LayerNorm/moments/mean"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Neg"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_175"
  op: "AddN"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_1_grad/Mul_1"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Mul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_1_grad/Mul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Mul"
  op: "Mul"
  input: "gradients/AddN_175"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Mul"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Sum"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Mul_1"
  op: "Mul"
  input: "electra/embeddings_1/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/AddN_175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Sum_1"
  op: "Sum"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Mul_1"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Sum_1/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 768
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape_1"
  op: "Reshape"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Sum_1"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  op: "RsqrtGrad"
  input: "electra/embeddings_1/LayerNorm/batchnorm/Rsqrt"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/add_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/add_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/add_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/batchnorm/add_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/add_grad/Sum"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/add_grad/Reshape/shape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/Rsqrt_grad/RsqrtGrad"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/Tile"
  op: "Tile"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/Reshape"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/Tile"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/scalar"
  op: "Const"
  input: "^gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/Mul"
  op: "Mul"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/scalar"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/sub"
  op: "Sub"
  input: "electra/embeddings_1/add_1"
  input: "electra/embeddings_1/LayerNorm/moments/StopGradient"
  input: "^gradients/electra/embeddings_1/LayerNorm/moments/variance_grad/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/mul_1"
  op: "Mul"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/Mul"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/Sum"
  op: "Sum"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/Sum"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/Neg"
  op: "Neg"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\000\002\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_2_grad/Reshape"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/Tile/multiples"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\001\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/Tile"
  op: "Tile"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/Reshape"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/Tile/multiples"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tmultiples"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 768.0
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/truediv"
  op: "RealDiv"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/Tile"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_176"
  op: "AddN"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_1_grad/Mul"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/SquaredDifference_grad/mul_1"
  input: "gradients/electra/embeddings_1/LayerNorm/moments/mean_grad/truediv"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_1_grad/Mul"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/add_1_grad/Sum/reduction_indices"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/add_1_grad/Sum"
  op: "Sum"
  input: "gradients/AddN_176"
  input: "gradients/electra/embeddings_1/add_1_grad/Sum/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/add_1_grad/Reshape/shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/add_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_1/add_1_grad/Sum"
  input: "gradients/electra/embeddings_1/add_1_grad/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/Reshape_2_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/Reshape_2_grad/Reshape"
  op: "Reshape"
  input: "gradients/electra/embeddings_1/add_1_grad/Reshape"
  input: "gradients/electra/embeddings_1/Reshape_2_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/AddN_176"
  input: "gradients/electra/embeddings/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/Reshape_1_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000@\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/Reshape_1_grad/Reshape"
  op: "Reshape"
  input: "gradients/AddN_176"
  input: "gradients/electra/embeddings_1/Reshape_1_grad/Shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/Slice_grad/Rank"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/Slice_grad/Shape"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/Slice_grad/zeros_like/shape_as_tensor"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/Slice_grad/zeros_like/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/Slice_grad/zeros_like"
  op: "Fill"
  input: "gradients/electra/embeddings_1/Slice_grad/zeros_like/shape_as_tensor"
  input: "gradients/electra/embeddings_1/Slice_grad/zeros_like/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/Slice_grad/XlaDynamicUpdateSlice"
  op: "XlaDynamicUpdateSlice"
  input: "gradients/electra/embeddings_1/Slice_grad/zeros_like"
  input: "gradients/electra/embeddings_1/Reshape_2_grad/Reshape"
  input: "electra/embeddings_1/Slice/begin"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/electra/embeddings/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/embeddings/Reshape_1_grad/Reshape"
  input: "electra/embeddings/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/embeddings/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/embeddings/one_hot"
  input: "gradients/electra/embeddings/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/MatMul_grad/MatMul"
  op: "MatMul"
  input: "gradients/electra/embeddings_1/Reshape_1_grad/Reshape"
  input: "electra/embeddings_1/MatMul/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 16384
          }
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: true
    }
  }
}
node {
  name: "gradients/electra/embeddings_1/MatMul_grad/MatMul_1"
  op: "MatMul"
  input: "electra/embeddings_1/one_hot"
  input: "gradients/electra/embeddings_1/Reshape_1_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: true
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "gradients/AddN_177"
  op: "AddN"
  input: "gradients/electra/embeddings_3/Slice_grad/XlaDynamicUpdateSlice"
  input: "gradients/electra/embeddings_1/Slice_grad/XlaDynamicUpdateSlice"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/Slice_grad/XlaDynamicUpdateSlice"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_178"
  op: "AddN"
  input: "gradients/generator_predictions/MatMul_grad/Reshape_1"
  input: "gradients/electra/embeddings_2/MatMul_grad/MatMul_1"
  input: "gradients/electra/embeddings/MatMul_grad/MatMul_1"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/MatMul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "gradients/AddN_179"
  op: "AddN"
  input: "gradients/electra/embeddings_3/MatMul_grad/MatMul_1"
  input: "gradients/electra/embeddings_1/MatMul_grad/MatMul_1"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss"
  op: "L2Loss"
  input: "gradients/AddN_178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/MatMul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_1"
  op: "L2Loss"
  input: "gradients/AddN_179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_2"
  op: "L2Loss"
  input: "gradients/AddN_177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/Slice_grad/XlaDynamicUpdateSlice"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_3"
  op: "L2Loss"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_4"
  op: "L2Loss"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_5"
  op: "L2Loss"
  input: "gradients/generator/embeddings_project/Tensordot/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/embeddings_project/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_6"
  op: "L2Loss"
  input: "gradients/generator/embeddings_project/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/embeddings_project/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_7"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_8"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_9"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_10"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_11"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_12"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_13"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_14"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_15"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_16"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_17"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_18"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_19"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_20"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_21"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_22"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_23"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_24"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_25"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_26"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_27"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_28"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_29"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_30"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_31"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_32"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_33"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_34"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_35"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_36"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_37"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_38"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_39"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_40"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_41"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_42"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_43"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_44"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_45"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_46"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_47"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_48"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_49"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_50"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_51"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_52"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_53"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_54"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_55"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_56"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_57"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_58"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_59"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_60"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_61"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_62"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_63"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_64"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_65"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_66"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_67"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_68"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_69"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_70"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_71"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_72"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_73"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_74"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_75"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_76"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_77"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_78"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_79"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_80"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_81"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_82"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_83"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_84"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_85"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_86"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_87"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_88"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_89"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_90"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_91"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_92"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_93"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_94"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_95"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_96"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_97"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_98"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_99"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_100"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_101"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_102"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_103"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_104"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_105"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_106"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_107"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_108"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_109"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_110"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_111"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_112"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_113"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_114"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_115"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_116"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_117"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_118"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_119"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_120"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_121"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_122"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_123"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_124"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_125"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_126"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_127"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_128"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_129"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_130"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_131"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_132"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_133"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_134"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_135"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_136"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_137"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_138"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_139"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_140"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_141"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_142"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_143"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_144"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_145"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_146"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_147"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_148"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_149"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_150"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_151"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_152"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_153"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_154"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_155"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_156"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_157"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_158"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_159"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_160"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_161"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_162"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_163"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_164"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_165"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_166"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_167"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_168"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_169"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_170"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_171"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_172"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_173"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_174"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_175"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_176"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_177"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_178"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_179"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_180"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_181"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_182"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_183"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_184"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_185"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_186"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_187"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_188"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_189"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_190"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_191"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_192"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_193"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_194"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_195"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_196"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_197"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_198"
  op: "L2Loss"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_199"
  op: "L2Loss"
  input: "gradients/generator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_200"
  op: "L2Loss"
  input: "gradients/generator_predictions/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_201"
  op: "L2Loss"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_202"
  op: "L2Loss"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_203"
  op: "L2Loss"
  input: "gradients/generator_predictions/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_204"
  op: "L2Loss"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_205"
  op: "L2Loss"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_206"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_207"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_208"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_209"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_210"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_211"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_212"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_213"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_214"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_215"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_216"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_217"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_218"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_219"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_220"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_221"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_222"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_223"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_224"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_225"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_226"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_227"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_228"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_229"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_230"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_231"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_232"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_233"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_234"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_235"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_236"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_237"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_238"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_239"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_240"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_241"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_242"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_243"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_244"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_245"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_246"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_247"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_248"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_249"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_250"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_251"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_252"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_253"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_254"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_255"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_256"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_257"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_258"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_259"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_260"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_261"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_262"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_263"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_264"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_265"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_266"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_267"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_268"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_269"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_270"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_271"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_272"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_273"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_274"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_275"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_276"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_277"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_278"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_279"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_280"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_281"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_282"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_283"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_284"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_285"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_286"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_287"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_288"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_289"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_290"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_291"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_292"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_293"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_294"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_295"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_296"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_297"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_298"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_299"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_300"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_301"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_302"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_303"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_304"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_305"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_306"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_307"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_308"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_309"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_310"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_311"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_312"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_313"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_314"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_315"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_316"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_317"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_318"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_319"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_320"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_321"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_322"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_323"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_324"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_325"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_326"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_327"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_328"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_329"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_330"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_331"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_332"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_333"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_334"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_335"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_336"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_337"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_338"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_339"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_340"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_341"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_342"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_343"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_344"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_345"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_346"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_347"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_348"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_349"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_350"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_351"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_352"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_353"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_354"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_355"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_356"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_357"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_358"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_359"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_360"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_361"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_362"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_363"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_364"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_365"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_366"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_367"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_368"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_369"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_370"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_371"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_372"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_373"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_374"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_375"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_376"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_377"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_378"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_379"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_380"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_381"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_382"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_383"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_384"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_385"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_386"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_387"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_388"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_389"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_390"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_391"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_392"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_393"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_394"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_395"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_396"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_397"
  op: "L2Loss"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_398"
  op: "L2Loss"
  input: "gradients/discriminator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_399"
  op: "L2Loss"
  input: "gradients/discriminator_predictions/dense/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_400"
  op: "L2Loss"
  input: "gradients/discriminator_predictions/dense_1/Tensordot/MatMul_grad/MatMul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense_1/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/L2Loss_401"
  op: "L2Loss"
  input: "gradients/discriminator_predictions/dense_1/BiasAdd_grad/BiasAddGrad"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense_1/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/stack"
  op: "Pack"
  input: "global_norm/L2Loss"
  input: "global_norm/L2Loss_1"
  input: "global_norm/L2Loss_2"
  input: "global_norm/L2Loss_3"
  input: "global_norm/L2Loss_4"
  input: "global_norm/L2Loss_5"
  input: "global_norm/L2Loss_6"
  input: "global_norm/L2Loss_7"
  input: "global_norm/L2Loss_8"
  input: "global_norm/L2Loss_9"
  input: "global_norm/L2Loss_10"
  input: "global_norm/L2Loss_11"
  input: "global_norm/L2Loss_12"
  input: "global_norm/L2Loss_13"
  input: "global_norm/L2Loss_14"
  input: "global_norm/L2Loss_15"
  input: "global_norm/L2Loss_16"
  input: "global_norm/L2Loss_17"
  input: "global_norm/L2Loss_18"
  input: "global_norm/L2Loss_19"
  input: "global_norm/L2Loss_20"
  input: "global_norm/L2Loss_21"
  input: "global_norm/L2Loss_22"
  input: "global_norm/L2Loss_23"
  input: "global_norm/L2Loss_24"
  input: "global_norm/L2Loss_25"
  input: "global_norm/L2Loss_26"
  input: "global_norm/L2Loss_27"
  input: "global_norm/L2Loss_28"
  input: "global_norm/L2Loss_29"
  input: "global_norm/L2Loss_30"
  input: "global_norm/L2Loss_31"
  input: "global_norm/L2Loss_32"
  input: "global_norm/L2Loss_33"
  input: "global_norm/L2Loss_34"
  input: "global_norm/L2Loss_35"
  input: "global_norm/L2Loss_36"
  input: "global_norm/L2Loss_37"
  input: "global_norm/L2Loss_38"
  input: "global_norm/L2Loss_39"
  input: "global_norm/L2Loss_40"
  input: "global_norm/L2Loss_41"
  input: "global_norm/L2Loss_42"
  input: "global_norm/L2Loss_43"
  input: "global_norm/L2Loss_44"
  input: "global_norm/L2Loss_45"
  input: "global_norm/L2Loss_46"
  input: "global_norm/L2Loss_47"
  input: "global_norm/L2Loss_48"
  input: "global_norm/L2Loss_49"
  input: "global_norm/L2Loss_50"
  input: "global_norm/L2Loss_51"
  input: "global_norm/L2Loss_52"
  input: "global_norm/L2Loss_53"
  input: "global_norm/L2Loss_54"
  input: "global_norm/L2Loss_55"
  input: "global_norm/L2Loss_56"
  input: "global_norm/L2Loss_57"
  input: "global_norm/L2Loss_58"
  input: "global_norm/L2Loss_59"
  input: "global_norm/L2Loss_60"
  input: "global_norm/L2Loss_61"
  input: "global_norm/L2Loss_62"
  input: "global_norm/L2Loss_63"
  input: "global_norm/L2Loss_64"
  input: "global_norm/L2Loss_65"
  input: "global_norm/L2Loss_66"
  input: "global_norm/L2Loss_67"
  input: "global_norm/L2Loss_68"
  input: "global_norm/L2Loss_69"
  input: "global_norm/L2Loss_70"
  input: "global_norm/L2Loss_71"
  input: "global_norm/L2Loss_72"
  input: "global_norm/L2Loss_73"
  input: "global_norm/L2Loss_74"
  input: "global_norm/L2Loss_75"
  input: "global_norm/L2Loss_76"
  input: "global_norm/L2Loss_77"
  input: "global_norm/L2Loss_78"
  input: "global_norm/L2Loss_79"
  input: "global_norm/L2Loss_80"
  input: "global_norm/L2Loss_81"
  input: "global_norm/L2Loss_82"
  input: "global_norm/L2Loss_83"
  input: "global_norm/L2Loss_84"
  input: "global_norm/L2Loss_85"
  input: "global_norm/L2Loss_86"
  input: "global_norm/L2Loss_87"
  input: "global_norm/L2Loss_88"
  input: "global_norm/L2Loss_89"
  input: "global_norm/L2Loss_90"
  input: "global_norm/L2Loss_91"
  input: "global_norm/L2Loss_92"
  input: "global_norm/L2Loss_93"
  input: "global_norm/L2Loss_94"
  input: "global_norm/L2Loss_95"
  input: "global_norm/L2Loss_96"
  input: "global_norm/L2Loss_97"
  input: "global_norm/L2Loss_98"
  input: "global_norm/L2Loss_99"
  input: "global_norm/L2Loss_100"
  input: "global_norm/L2Loss_101"
  input: "global_norm/L2Loss_102"
  input: "global_norm/L2Loss_103"
  input: "global_norm/L2Loss_104"
  input: "global_norm/L2Loss_105"
  input: "global_norm/L2Loss_106"
  input: "global_norm/L2Loss_107"
  input: "global_norm/L2Loss_108"
  input: "global_norm/L2Loss_109"
  input: "global_norm/L2Loss_110"
  input: "global_norm/L2Loss_111"
  input: "global_norm/L2Loss_112"
  input: "global_norm/L2Loss_113"
  input: "global_norm/L2Loss_114"
  input: "global_norm/L2Loss_115"
  input: "global_norm/L2Loss_116"
  input: "global_norm/L2Loss_117"
  input: "global_norm/L2Loss_118"
  input: "global_norm/L2Loss_119"
  input: "global_norm/L2Loss_120"
  input: "global_norm/L2Loss_121"
  input: "global_norm/L2Loss_122"
  input: "global_norm/L2Loss_123"
  input: "global_norm/L2Loss_124"
  input: "global_norm/L2Loss_125"
  input: "global_norm/L2Loss_126"
  input: "global_norm/L2Loss_127"
  input: "global_norm/L2Loss_128"
  input: "global_norm/L2Loss_129"
  input: "global_norm/L2Loss_130"
  input: "global_norm/L2Loss_131"
  input: "global_norm/L2Loss_132"
  input: "global_norm/L2Loss_133"
  input: "global_norm/L2Loss_134"
  input: "global_norm/L2Loss_135"
  input: "global_norm/L2Loss_136"
  input: "global_norm/L2Loss_137"
  input: "global_norm/L2Loss_138"
  input: "global_norm/L2Loss_139"
  input: "global_norm/L2Loss_140"
  input: "global_norm/L2Loss_141"
  input: "global_norm/L2Loss_142"
  input: "global_norm/L2Loss_143"
  input: "global_norm/L2Loss_144"
  input: "global_norm/L2Loss_145"
  input: "global_norm/L2Loss_146"
  input: "global_norm/L2Loss_147"
  input: "global_norm/L2Loss_148"
  input: "global_norm/L2Loss_149"
  input: "global_norm/L2Loss_150"
  input: "global_norm/L2Loss_151"
  input: "global_norm/L2Loss_152"
  input: "global_norm/L2Loss_153"
  input: "global_norm/L2Loss_154"
  input: "global_norm/L2Loss_155"
  input: "global_norm/L2Loss_156"
  input: "global_norm/L2Loss_157"
  input: "global_norm/L2Loss_158"
  input: "global_norm/L2Loss_159"
  input: "global_norm/L2Loss_160"
  input: "global_norm/L2Loss_161"
  input: "global_norm/L2Loss_162"
  input: "global_norm/L2Loss_163"
  input: "global_norm/L2Loss_164"
  input: "global_norm/L2Loss_165"
  input: "global_norm/L2Loss_166"
  input: "global_norm/L2Loss_167"
  input: "global_norm/L2Loss_168"
  input: "global_norm/L2Loss_169"
  input: "global_norm/L2Loss_170"
  input: "global_norm/L2Loss_171"
  input: "global_norm/L2Loss_172"
  input: "global_norm/L2Loss_173"
  input: "global_norm/L2Loss_174"
  input: "global_norm/L2Loss_175"
  input: "global_norm/L2Loss_176"
  input: "global_norm/L2Loss_177"
  input: "global_norm/L2Loss_178"
  input: "global_norm/L2Loss_179"
  input: "global_norm/L2Loss_180"
  input: "global_norm/L2Loss_181"
  input: "global_norm/L2Loss_182"
  input: "global_norm/L2Loss_183"
  input: "global_norm/L2Loss_184"
  input: "global_norm/L2Loss_185"
  input: "global_norm/L2Loss_186"
  input: "global_norm/L2Loss_187"
  input: "global_norm/L2Loss_188"
  input: "global_norm/L2Loss_189"
  input: "global_norm/L2Loss_190"
  input: "global_norm/L2Loss_191"
  input: "global_norm/L2Loss_192"
  input: "global_norm/L2Loss_193"
  input: "global_norm/L2Loss_194"
  input: "global_norm/L2Loss_195"
  input: "global_norm/L2Loss_196"
  input: "global_norm/L2Loss_197"
  input: "global_norm/L2Loss_198"
  input: "global_norm/L2Loss_199"
  input: "global_norm/L2Loss_200"
  input: "global_norm/L2Loss_201"
  input: "global_norm/L2Loss_202"
  input: "global_norm/L2Loss_203"
  input: "global_norm/L2Loss_204"
  input: "global_norm/L2Loss_205"
  input: "global_norm/L2Loss_206"
  input: "global_norm/L2Loss_207"
  input: "global_norm/L2Loss_208"
  input: "global_norm/L2Loss_209"
  input: "global_norm/L2Loss_210"
  input: "global_norm/L2Loss_211"
  input: "global_norm/L2Loss_212"
  input: "global_norm/L2Loss_213"
  input: "global_norm/L2Loss_214"
  input: "global_norm/L2Loss_215"
  input: "global_norm/L2Loss_216"
  input: "global_norm/L2Loss_217"
  input: "global_norm/L2Loss_218"
  input: "global_norm/L2Loss_219"
  input: "global_norm/L2Loss_220"
  input: "global_norm/L2Loss_221"
  input: "global_norm/L2Loss_222"
  input: "global_norm/L2Loss_223"
  input: "global_norm/L2Loss_224"
  input: "global_norm/L2Loss_225"
  input: "global_norm/L2Loss_226"
  input: "global_norm/L2Loss_227"
  input: "global_norm/L2Loss_228"
  input: "global_norm/L2Loss_229"
  input: "global_norm/L2Loss_230"
  input: "global_norm/L2Loss_231"
  input: "global_norm/L2Loss_232"
  input: "global_norm/L2Loss_233"
  input: "global_norm/L2Loss_234"
  input: "global_norm/L2Loss_235"
  input: "global_norm/L2Loss_236"
  input: "global_norm/L2Loss_237"
  input: "global_norm/L2Loss_238"
  input: "global_norm/L2Loss_239"
  input: "global_norm/L2Loss_240"
  input: "global_norm/L2Loss_241"
  input: "global_norm/L2Loss_242"
  input: "global_norm/L2Loss_243"
  input: "global_norm/L2Loss_244"
  input: "global_norm/L2Loss_245"
  input: "global_norm/L2Loss_246"
  input: "global_norm/L2Loss_247"
  input: "global_norm/L2Loss_248"
  input: "global_norm/L2Loss_249"
  input: "global_norm/L2Loss_250"
  input: "global_norm/L2Loss_251"
  input: "global_norm/L2Loss_252"
  input: "global_norm/L2Loss_253"
  input: "global_norm/L2Loss_254"
  input: "global_norm/L2Loss_255"
  input: "global_norm/L2Loss_256"
  input: "global_norm/L2Loss_257"
  input: "global_norm/L2Loss_258"
  input: "global_norm/L2Loss_259"
  input: "global_norm/L2Loss_260"
  input: "global_norm/L2Loss_261"
  input: "global_norm/L2Loss_262"
  input: "global_norm/L2Loss_263"
  input: "global_norm/L2Loss_264"
  input: "global_norm/L2Loss_265"
  input: "global_norm/L2Loss_266"
  input: "global_norm/L2Loss_267"
  input: "global_norm/L2Loss_268"
  input: "global_norm/L2Loss_269"
  input: "global_norm/L2Loss_270"
  input: "global_norm/L2Loss_271"
  input: "global_norm/L2Loss_272"
  input: "global_norm/L2Loss_273"
  input: "global_norm/L2Loss_274"
  input: "global_norm/L2Loss_275"
  input: "global_norm/L2Loss_276"
  input: "global_norm/L2Loss_277"
  input: "global_norm/L2Loss_278"
  input: "global_norm/L2Loss_279"
  input: "global_norm/L2Loss_280"
  input: "global_norm/L2Loss_281"
  input: "global_norm/L2Loss_282"
  input: "global_norm/L2Loss_283"
  input: "global_norm/L2Loss_284"
  input: "global_norm/L2Loss_285"
  input: "global_norm/L2Loss_286"
  input: "global_norm/L2Loss_287"
  input: "global_norm/L2Loss_288"
  input: "global_norm/L2Loss_289"
  input: "global_norm/L2Loss_290"
  input: "global_norm/L2Loss_291"
  input: "global_norm/L2Loss_292"
  input: "global_norm/L2Loss_293"
  input: "global_norm/L2Loss_294"
  input: "global_norm/L2Loss_295"
  input: "global_norm/L2Loss_296"
  input: "global_norm/L2Loss_297"
  input: "global_norm/L2Loss_298"
  input: "global_norm/L2Loss_299"
  input: "global_norm/L2Loss_300"
  input: "global_norm/L2Loss_301"
  input: "global_norm/L2Loss_302"
  input: "global_norm/L2Loss_303"
  input: "global_norm/L2Loss_304"
  input: "global_norm/L2Loss_305"
  input: "global_norm/L2Loss_306"
  input: "global_norm/L2Loss_307"
  input: "global_norm/L2Loss_308"
  input: "global_norm/L2Loss_309"
  input: "global_norm/L2Loss_310"
  input: "global_norm/L2Loss_311"
  input: "global_norm/L2Loss_312"
  input: "global_norm/L2Loss_313"
  input: "global_norm/L2Loss_314"
  input: "global_norm/L2Loss_315"
  input: "global_norm/L2Loss_316"
  input: "global_norm/L2Loss_317"
  input: "global_norm/L2Loss_318"
  input: "global_norm/L2Loss_319"
  input: "global_norm/L2Loss_320"
  input: "global_norm/L2Loss_321"
  input: "global_norm/L2Loss_322"
  input: "global_norm/L2Loss_323"
  input: "global_norm/L2Loss_324"
  input: "global_norm/L2Loss_325"
  input: "global_norm/L2Loss_326"
  input: "global_norm/L2Loss_327"
  input: "global_norm/L2Loss_328"
  input: "global_norm/L2Loss_329"
  input: "global_norm/L2Loss_330"
  input: "global_norm/L2Loss_331"
  input: "global_norm/L2Loss_332"
  input: "global_norm/L2Loss_333"
  input: "global_norm/L2Loss_334"
  input: "global_norm/L2Loss_335"
  input: "global_norm/L2Loss_336"
  input: "global_norm/L2Loss_337"
  input: "global_norm/L2Loss_338"
  input: "global_norm/L2Loss_339"
  input: "global_norm/L2Loss_340"
  input: "global_norm/L2Loss_341"
  input: "global_norm/L2Loss_342"
  input: "global_norm/L2Loss_343"
  input: "global_norm/L2Loss_344"
  input: "global_norm/L2Loss_345"
  input: "global_norm/L2Loss_346"
  input: "global_norm/L2Loss_347"
  input: "global_norm/L2Loss_348"
  input: "global_norm/L2Loss_349"
  input: "global_norm/L2Loss_350"
  input: "global_norm/L2Loss_351"
  input: "global_norm/L2Loss_352"
  input: "global_norm/L2Loss_353"
  input: "global_norm/L2Loss_354"
  input: "global_norm/L2Loss_355"
  input: "global_norm/L2Loss_356"
  input: "global_norm/L2Loss_357"
  input: "global_norm/L2Loss_358"
  input: "global_norm/L2Loss_359"
  input: "global_norm/L2Loss_360"
  input: "global_norm/L2Loss_361"
  input: "global_norm/L2Loss_362"
  input: "global_norm/L2Loss_363"
  input: "global_norm/L2Loss_364"
  input: "global_norm/L2Loss_365"
  input: "global_norm/L2Loss_366"
  input: "global_norm/L2Loss_367"
  input: "global_norm/L2Loss_368"
  input: "global_norm/L2Loss_369"
  input: "global_norm/L2Loss_370"
  input: "global_norm/L2Loss_371"
  input: "global_norm/L2Loss_372"
  input: "global_norm/L2Loss_373"
  input: "global_norm/L2Loss_374"
  input: "global_norm/L2Loss_375"
  input: "global_norm/L2Loss_376"
  input: "global_norm/L2Loss_377"
  input: "global_norm/L2Loss_378"
  input: "global_norm/L2Loss_379"
  input: "global_norm/L2Loss_380"
  input: "global_norm/L2Loss_381"
  input: "global_norm/L2Loss_382"
  input: "global_norm/L2Loss_383"
  input: "global_norm/L2Loss_384"
  input: "global_norm/L2Loss_385"
  input: "global_norm/L2Loss_386"
  input: "global_norm/L2Loss_387"
  input: "global_norm/L2Loss_388"
  input: "global_norm/L2Loss_389"
  input: "global_norm/L2Loss_390"
  input: "global_norm/L2Loss_391"
  input: "global_norm/L2Loss_392"
  input: "global_norm/L2Loss_393"
  input: "global_norm/L2Loss_394"
  input: "global_norm/L2Loss_395"
  input: "global_norm/L2Loss_396"
  input: "global_norm/L2Loss_397"
  input: "global_norm/L2Loss_398"
  input: "global_norm/L2Loss_399"
  input: "global_norm/L2Loss_400"
  input: "global_norm/L2Loss_401"
  attr {
    key: "N"
    value {
      i: 402
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 402
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
}
node {
  name: "global_norm/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "global_norm/Sum"
  op: "Sum"
  input: "global_norm/stack"
  input: "global_norm/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "global_norm/Const_1"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "global_norm/mul"
  op: "Mul"
  input: "global_norm/Sum"
  input: "global_norm/Const_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "global_norm/global_norm"
  op: "Sqrt"
  input: "global_norm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/truediv/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "clip_by_global_norm/truediv"
  op: "RealDiv"
  input: "clip_by_global_norm/truediv/x"
  input: "global_norm/global_norm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/Const"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "clip_by_global_norm/truediv_1/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "clip_by_global_norm/truediv_1"
  op: "RealDiv"
  input: "clip_by_global_norm/Const"
  input: "clip_by_global_norm/truediv_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/Minimum"
  op: "Minimum"
  input: "clip_by_global_norm/truediv"
  input: "clip_by_global_norm/truediv_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "clip_by_global_norm/mul"
  op: "Mul"
  input: "clip_by_global_norm/mul/x"
  input: "clip_by_global_norm/Minimum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/sub"
  op: "Sub"
  input: "global_norm/global_norm"
  input: "global_norm/global_norm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/add"
  op: "AddV2"
  input: "clip_by_global_norm/mul"
  input: "clip_by_global_norm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_1"
  op: "Mul"
  input: "gradients/AddN_178"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/MatMul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_0"
  op: "Identity"
  input: "clip_by_global_norm/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/MatMul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_2"
  op: "Mul"
  input: "gradients/AddN_179"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_1"
  op: "Identity"
  input: "clip_by_global_norm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_3"
  op: "Mul"
  input: "gradients/AddN_177"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/Slice_grad/XlaDynamicUpdateSlice"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_2"
  op: "Identity"
  input: "clip_by_global_norm/mul_3"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/Slice_grad/XlaDynamicUpdateSlice"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_4"
  op: "Mul"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_3"
  op: "Identity"
  input: "clip_by_global_norm/mul_4"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_5"
  op: "Mul"
  input: "gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_4"
  op: "Identity"
  input: "clip_by_global_norm/mul_5"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_6"
  op: "Mul"
  input: "gradients/generator/embeddings_project/Tensordot/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/embeddings_project/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_5"
  op: "Identity"
  input: "clip_by_global_norm/mul_6"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/embeddings_project/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_7"
  op: "Mul"
  input: "gradients/generator/embeddings_project/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/embeddings_project/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_6"
  op: "Identity"
  input: "clip_by_global_norm/mul_7"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/embeddings_project/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_8"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_7"
  op: "Identity"
  input: "clip_by_global_norm/mul_8"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_9"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_8"
  op: "Identity"
  input: "clip_by_global_norm/mul_9"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_10"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_9"
  op: "Identity"
  input: "clip_by_global_norm/mul_10"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_11"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_10"
  op: "Identity"
  input: "clip_by_global_norm/mul_11"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_12"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_11"
  op: "Identity"
  input: "clip_by_global_norm/mul_12"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_13"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_12"
  op: "Identity"
  input: "clip_by_global_norm/mul_13"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_14"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_13"
  op: "Identity"
  input: "clip_by_global_norm/mul_14"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_15"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_14"
  op: "Identity"
  input: "clip_by_global_norm/mul_15"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_16"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_15"
  op: "Identity"
  input: "clip_by_global_norm/mul_16"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_17"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_16"
  op: "Identity"
  input: "clip_by_global_norm/mul_17"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_18"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_17"
  op: "Identity"
  input: "clip_by_global_norm/mul_18"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_19"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_18"
  op: "Identity"
  input: "clip_by_global_norm/mul_19"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_20"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_19"
  op: "Identity"
  input: "clip_by_global_norm/mul_20"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_21"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_20"
  op: "Identity"
  input: "clip_by_global_norm/mul_21"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_22"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_21"
  op: "Identity"
  input: "clip_by_global_norm/mul_22"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_23"
  op: "Mul"
  input: "gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_22"
  op: "Identity"
  input: "clip_by_global_norm/mul_23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_24"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_23"
  op: "Identity"
  input: "clip_by_global_norm/mul_24"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_25"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_24"
  op: "Identity"
  input: "clip_by_global_norm/mul_25"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_26"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_25"
  op: "Identity"
  input: "clip_by_global_norm/mul_26"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_27"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_26"
  op: "Identity"
  input: "clip_by_global_norm/mul_27"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_28"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_27"
  op: "Identity"
  input: "clip_by_global_norm/mul_28"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_29"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_28"
  op: "Identity"
  input: "clip_by_global_norm/mul_29"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_30"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_29"
  op: "Identity"
  input: "clip_by_global_norm/mul_30"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_31"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_30"
  op: "Identity"
  input: "clip_by_global_norm/mul_31"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_32"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_31"
  op: "Identity"
  input: "clip_by_global_norm/mul_32"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_33"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_32"
  op: "Identity"
  input: "clip_by_global_norm/mul_33"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_34"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_33"
  op: "Identity"
  input: "clip_by_global_norm/mul_34"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_35"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_34"
  op: "Identity"
  input: "clip_by_global_norm/mul_35"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_36"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_35"
  op: "Identity"
  input: "clip_by_global_norm/mul_36"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_37"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_36"
  op: "Identity"
  input: "clip_by_global_norm/mul_37"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_38"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_37"
  op: "Identity"
  input: "clip_by_global_norm/mul_38"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_39"
  op: "Mul"
  input: "gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_38"
  op: "Identity"
  input: "clip_by_global_norm/mul_39"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_40"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_39"
  op: "Identity"
  input: "clip_by_global_norm/mul_40"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_41"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_40"
  op: "Identity"
  input: "clip_by_global_norm/mul_41"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_42"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_41"
  op: "Identity"
  input: "clip_by_global_norm/mul_42"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_43"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_42"
  op: "Identity"
  input: "clip_by_global_norm/mul_43"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_44"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_43"
  op: "Identity"
  input: "clip_by_global_norm/mul_44"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_45"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_44"
  op: "Identity"
  input: "clip_by_global_norm/mul_45"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_46"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_45"
  op: "Identity"
  input: "clip_by_global_norm/mul_46"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_47"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_46"
  op: "Identity"
  input: "clip_by_global_norm/mul_47"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_48"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_47"
  op: "Identity"
  input: "clip_by_global_norm/mul_48"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_49"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_48"
  op: "Identity"
  input: "clip_by_global_norm/mul_49"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_50"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_49"
  op: "Identity"
  input: "clip_by_global_norm/mul_50"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_51"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_50"
  op: "Identity"
  input: "clip_by_global_norm/mul_51"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_52"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_51"
  op: "Identity"
  input: "clip_by_global_norm/mul_52"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_53"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_52"
  op: "Identity"
  input: "clip_by_global_norm/mul_53"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_54"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_53"
  op: "Identity"
  input: "clip_by_global_norm/mul_54"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_55"
  op: "Mul"
  input: "gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_54"
  op: "Identity"
  input: "clip_by_global_norm/mul_55"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_56"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_55"
  op: "Identity"
  input: "clip_by_global_norm/mul_56"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_57"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_56"
  op: "Identity"
  input: "clip_by_global_norm/mul_57"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_58"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_57"
  op: "Identity"
  input: "clip_by_global_norm/mul_58"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_59"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_58"
  op: "Identity"
  input: "clip_by_global_norm/mul_59"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_60"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_59"
  op: "Identity"
  input: "clip_by_global_norm/mul_60"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_61"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_60"
  op: "Identity"
  input: "clip_by_global_norm/mul_61"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_62"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_61"
  op: "Identity"
  input: "clip_by_global_norm/mul_62"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_63"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_62"
  op: "Identity"
  input: "clip_by_global_norm/mul_63"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_64"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_63"
  op: "Identity"
  input: "clip_by_global_norm/mul_64"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_65"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_64"
  op: "Identity"
  input: "clip_by_global_norm/mul_65"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_66"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_65"
  op: "Identity"
  input: "clip_by_global_norm/mul_66"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_67"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_66"
  op: "Identity"
  input: "clip_by_global_norm/mul_67"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_68"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_67"
  op: "Identity"
  input: "clip_by_global_norm/mul_68"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_69"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_68"
  op: "Identity"
  input: "clip_by_global_norm/mul_69"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_70"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_69"
  op: "Identity"
  input: "clip_by_global_norm/mul_70"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_71"
  op: "Mul"
  input: "gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_70"
  op: "Identity"
  input: "clip_by_global_norm/mul_71"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_72"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_71"
  op: "Identity"
  input: "clip_by_global_norm/mul_72"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_73"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_72"
  op: "Identity"
  input: "clip_by_global_norm/mul_73"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_74"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_73"
  op: "Identity"
  input: "clip_by_global_norm/mul_74"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_75"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_74"
  op: "Identity"
  input: "clip_by_global_norm/mul_75"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_76"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_75"
  op: "Identity"
  input: "clip_by_global_norm/mul_76"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_77"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_76"
  op: "Identity"
  input: "clip_by_global_norm/mul_77"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_78"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_77"
  op: "Identity"
  input: "clip_by_global_norm/mul_78"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_79"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_78"
  op: "Identity"
  input: "clip_by_global_norm/mul_79"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_80"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_79"
  op: "Identity"
  input: "clip_by_global_norm/mul_80"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_81"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_80"
  op: "Identity"
  input: "clip_by_global_norm/mul_81"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_82"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_81"
  op: "Identity"
  input: "clip_by_global_norm/mul_82"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_83"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_82"
  op: "Identity"
  input: "clip_by_global_norm/mul_83"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_84"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_83"
  op: "Identity"
  input: "clip_by_global_norm/mul_84"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_85"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_84"
  op: "Identity"
  input: "clip_by_global_norm/mul_85"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_86"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_85"
  op: "Identity"
  input: "clip_by_global_norm/mul_86"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_87"
  op: "Mul"
  input: "gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_86"
  op: "Identity"
  input: "clip_by_global_norm/mul_87"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_88"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_87"
  op: "Identity"
  input: "clip_by_global_norm/mul_88"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_89"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_88"
  op: "Identity"
  input: "clip_by_global_norm/mul_89"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_90"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_89"
  op: "Identity"
  input: "clip_by_global_norm/mul_90"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_91"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_90"
  op: "Identity"
  input: "clip_by_global_norm/mul_91"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_92"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_91"
  op: "Identity"
  input: "clip_by_global_norm/mul_92"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_93"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_92"
  op: "Identity"
  input: "clip_by_global_norm/mul_93"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_94"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_93"
  op: "Identity"
  input: "clip_by_global_norm/mul_94"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_95"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_94"
  op: "Identity"
  input: "clip_by_global_norm/mul_95"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_96"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_95"
  op: "Identity"
  input: "clip_by_global_norm/mul_96"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_97"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_96"
  op: "Identity"
  input: "clip_by_global_norm/mul_97"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_98"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_97"
  op: "Identity"
  input: "clip_by_global_norm/mul_98"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_99"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_98"
  op: "Identity"
  input: "clip_by_global_norm/mul_99"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_100"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_99"
  op: "Identity"
  input: "clip_by_global_norm/mul_100"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_101"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_100"
  op: "Identity"
  input: "clip_by_global_norm/mul_101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_102"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_101"
  op: "Identity"
  input: "clip_by_global_norm/mul_102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_103"
  op: "Mul"
  input: "gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_102"
  op: "Identity"
  input: "clip_by_global_norm/mul_103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_104"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_103"
  op: "Identity"
  input: "clip_by_global_norm/mul_104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_105"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_104"
  op: "Identity"
  input: "clip_by_global_norm/mul_105"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_106"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_105"
  op: "Identity"
  input: "clip_by_global_norm/mul_106"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_107"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_106"
  op: "Identity"
  input: "clip_by_global_norm/mul_107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_108"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_107"
  op: "Identity"
  input: "clip_by_global_norm/mul_108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_109"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_108"
  op: "Identity"
  input: "clip_by_global_norm/mul_109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_110"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_109"
  op: "Identity"
  input: "clip_by_global_norm/mul_110"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_111"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_110"
  op: "Identity"
  input: "clip_by_global_norm/mul_111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_112"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_111"
  op: "Identity"
  input: "clip_by_global_norm/mul_112"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_113"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_112"
  op: "Identity"
  input: "clip_by_global_norm/mul_113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_114"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_113"
  op: "Identity"
  input: "clip_by_global_norm/mul_114"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_115"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_114"
  op: "Identity"
  input: "clip_by_global_norm/mul_115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_116"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_115"
  op: "Identity"
  input: "clip_by_global_norm/mul_116"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_117"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_116"
  op: "Identity"
  input: "clip_by_global_norm/mul_117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_118"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_117"
  op: "Identity"
  input: "clip_by_global_norm/mul_118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_119"
  op: "Mul"
  input: "gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_118"
  op: "Identity"
  input: "clip_by_global_norm/mul_119"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_120"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_119"
  op: "Identity"
  input: "clip_by_global_norm/mul_120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_121"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_120"
  op: "Identity"
  input: "clip_by_global_norm/mul_121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_122"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_121"
  op: "Identity"
  input: "clip_by_global_norm/mul_122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_123"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_122"
  op: "Identity"
  input: "clip_by_global_norm/mul_123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_124"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_123"
  op: "Identity"
  input: "clip_by_global_norm/mul_124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_125"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_124"
  op: "Identity"
  input: "clip_by_global_norm/mul_125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_126"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_125"
  op: "Identity"
  input: "clip_by_global_norm/mul_126"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_127"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_126"
  op: "Identity"
  input: "clip_by_global_norm/mul_127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_128"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_127"
  op: "Identity"
  input: "clip_by_global_norm/mul_128"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_129"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_128"
  op: "Identity"
  input: "clip_by_global_norm/mul_129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_130"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_129"
  op: "Identity"
  input: "clip_by_global_norm/mul_130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_131"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_130"
  op: "Identity"
  input: "clip_by_global_norm/mul_131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_132"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_131"
  op: "Identity"
  input: "clip_by_global_norm/mul_132"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_133"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_132"
  op: "Identity"
  input: "clip_by_global_norm/mul_133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_134"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_133"
  op: "Identity"
  input: "clip_by_global_norm/mul_134"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_135"
  op: "Mul"
  input: "gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_134"
  op: "Identity"
  input: "clip_by_global_norm/mul_135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_136"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_135"
  op: "Identity"
  input: "clip_by_global_norm/mul_136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_137"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_136"
  op: "Identity"
  input: "clip_by_global_norm/mul_137"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_138"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_137"
  op: "Identity"
  input: "clip_by_global_norm/mul_138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_139"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_138"
  op: "Identity"
  input: "clip_by_global_norm/mul_139"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_140"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_139"
  op: "Identity"
  input: "clip_by_global_norm/mul_140"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_141"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_140"
  op: "Identity"
  input: "clip_by_global_norm/mul_141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_142"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_141"
  op: "Identity"
  input: "clip_by_global_norm/mul_142"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_143"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_142"
  op: "Identity"
  input: "clip_by_global_norm/mul_143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_144"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_143"
  op: "Identity"
  input: "clip_by_global_norm/mul_144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_145"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_144"
  op: "Identity"
  input: "clip_by_global_norm/mul_145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_146"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_145"
  op: "Identity"
  input: "clip_by_global_norm/mul_146"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_147"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_146"
  op: "Identity"
  input: "clip_by_global_norm/mul_147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_148"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_147"
  op: "Identity"
  input: "clip_by_global_norm/mul_148"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_149"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_148"
  op: "Identity"
  input: "clip_by_global_norm/mul_149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_150"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_149"
  op: "Identity"
  input: "clip_by_global_norm/mul_150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_151"
  op: "Mul"
  input: "gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_150"
  op: "Identity"
  input: "clip_by_global_norm/mul_151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_152"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_151"
  op: "Identity"
  input: "clip_by_global_norm/mul_152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_153"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_152"
  op: "Identity"
  input: "clip_by_global_norm/mul_153"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_154"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_153"
  op: "Identity"
  input: "clip_by_global_norm/mul_154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_155"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_154"
  op: "Identity"
  input: "clip_by_global_norm/mul_155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_156"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_155"
  op: "Identity"
  input: "clip_by_global_norm/mul_156"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_157"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_156"
  op: "Identity"
  input: "clip_by_global_norm/mul_157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_158"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_157"
  op: "Identity"
  input: "clip_by_global_norm/mul_158"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_159"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_158"
  op: "Identity"
  input: "clip_by_global_norm/mul_159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_160"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_159"
  op: "Identity"
  input: "clip_by_global_norm/mul_160"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_161"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_160"
  op: "Identity"
  input: "clip_by_global_norm/mul_161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_162"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_161"
  op: "Identity"
  input: "clip_by_global_norm/mul_162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_163"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_162"
  op: "Identity"
  input: "clip_by_global_norm/mul_163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_164"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_163"
  op: "Identity"
  input: "clip_by_global_norm/mul_164"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_165"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_164"
  op: "Identity"
  input: "clip_by_global_norm/mul_165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_166"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_165"
  op: "Identity"
  input: "clip_by_global_norm/mul_166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_167"
  op: "Mul"
  input: "gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_166"
  op: "Identity"
  input: "clip_by_global_norm/mul_167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_168"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_167"
  op: "Identity"
  input: "clip_by_global_norm/mul_168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_169"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_168"
  op: "Identity"
  input: "clip_by_global_norm/mul_169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_170"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_169"
  op: "Identity"
  input: "clip_by_global_norm/mul_170"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_171"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_170"
  op: "Identity"
  input: "clip_by_global_norm/mul_171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_172"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_171"
  op: "Identity"
  input: "clip_by_global_norm/mul_172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_173"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_172"
  op: "Identity"
  input: "clip_by_global_norm/mul_173"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_174"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_173"
  op: "Identity"
  input: "clip_by_global_norm/mul_174"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_175"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_174"
  op: "Identity"
  input: "clip_by_global_norm/mul_175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_176"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_175"
  op: "Identity"
  input: "clip_by_global_norm/mul_176"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_177"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_176"
  op: "Identity"
  input: "clip_by_global_norm/mul_177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_178"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_177"
  op: "Identity"
  input: "clip_by_global_norm/mul_178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_179"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_178"
  op: "Identity"
  input: "clip_by_global_norm/mul_179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_180"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_179"
  op: "Identity"
  input: "clip_by_global_norm/mul_180"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_181"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_180"
  op: "Identity"
  input: "clip_by_global_norm/mul_181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_182"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_181"
  op: "Identity"
  input: "clip_by_global_norm/mul_182"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_183"
  op: "Mul"
  input: "gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_182"
  op: "Identity"
  input: "clip_by_global_norm/mul_183"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_184"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_183"
  op: "Identity"
  input: "clip_by_global_norm/mul_184"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_185"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_184"
  op: "Identity"
  input: "clip_by_global_norm/mul_185"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_186"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_185"
  op: "Identity"
  input: "clip_by_global_norm/mul_186"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_187"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_186"
  op: "Identity"
  input: "clip_by_global_norm/mul_187"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_188"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_187"
  op: "Identity"
  input: "clip_by_global_norm/mul_188"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_189"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_188"
  op: "Identity"
  input: "clip_by_global_norm/mul_189"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_190"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_189"
  op: "Identity"
  input: "clip_by_global_norm/mul_190"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_191"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_190"
  op: "Identity"
  input: "clip_by_global_norm/mul_191"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_192"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_191"
  op: "Identity"
  input: "clip_by_global_norm/mul_192"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_193"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_192"
  op: "Identity"
  input: "clip_by_global_norm/mul_193"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_194"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_193"
  op: "Identity"
  input: "clip_by_global_norm/mul_194"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_195"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_194"
  op: "Identity"
  input: "clip_by_global_norm/mul_195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_196"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_195"
  op: "Identity"
  input: "clip_by_global_norm/mul_196"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_197"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_196"
  op: "Identity"
  input: "clip_by_global_norm/mul_197"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_198"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_197"
  op: "Identity"
  input: "clip_by_global_norm/mul_198"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_199"
  op: "Mul"
  input: "gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_198"
  op: "Identity"
  input: "clip_by_global_norm/mul_199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_200"
  op: "Mul"
  input: "gradients/generator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_199"
  op: "Identity"
  input: "clip_by_global_norm/mul_200"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_201"
  op: "Mul"
  input: "gradients/generator_predictions/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_200"
  op: "Identity"
  input: "clip_by_global_norm/mul_201"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_202"
  op: "Mul"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_201"
  op: "Identity"
  input: "clip_by_global_norm/mul_202"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_203"
  op: "Mul"
  input: "gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_202"
  op: "Identity"
  input: "clip_by_global_norm/mul_203"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_204"
  op: "Mul"
  input: "gradients/generator_predictions/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_203"
  op: "Identity"
  input: "clip_by_global_norm/mul_204"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_205"
  op: "Mul"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_204"
  op: "Identity"
  input: "clip_by_global_norm/mul_205"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_206"
  op: "Mul"
  input: "gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_205"
  op: "Identity"
  input: "clip_by_global_norm/mul_206"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_207"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_206"
  op: "Identity"
  input: "clip_by_global_norm/mul_207"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_208"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_207"
  op: "Identity"
  input: "clip_by_global_norm/mul_208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_209"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_208"
  op: "Identity"
  input: "clip_by_global_norm/mul_209"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_210"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_209"
  op: "Identity"
  input: "clip_by_global_norm/mul_210"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_211"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_210"
  op: "Identity"
  input: "clip_by_global_norm/mul_211"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_212"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_211"
  op: "Identity"
  input: "clip_by_global_norm/mul_212"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_213"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_212"
  op: "Identity"
  input: "clip_by_global_norm/mul_213"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_214"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_213"
  op: "Identity"
  input: "clip_by_global_norm/mul_214"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_215"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_214"
  op: "Identity"
  input: "clip_by_global_norm/mul_215"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_216"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_215"
  op: "Identity"
  input: "clip_by_global_norm/mul_216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_217"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_216"
  op: "Identity"
  input: "clip_by_global_norm/mul_217"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_218"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_217"
  op: "Identity"
  input: "clip_by_global_norm/mul_218"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_219"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_218"
  op: "Identity"
  input: "clip_by_global_norm/mul_219"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_220"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_219"
  op: "Identity"
  input: "clip_by_global_norm/mul_220"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_221"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_220"
  op: "Identity"
  input: "clip_by_global_norm/mul_221"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_222"
  op: "Mul"
  input: "gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_221"
  op: "Identity"
  input: "clip_by_global_norm/mul_222"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_223"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_222"
  op: "Identity"
  input: "clip_by_global_norm/mul_223"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_224"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_223"
  op: "Identity"
  input: "clip_by_global_norm/mul_224"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_225"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_224"
  op: "Identity"
  input: "clip_by_global_norm/mul_225"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_226"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_225"
  op: "Identity"
  input: "clip_by_global_norm/mul_226"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_227"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_226"
  op: "Identity"
  input: "clip_by_global_norm/mul_227"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_228"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_227"
  op: "Identity"
  input: "clip_by_global_norm/mul_228"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_229"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_228"
  op: "Identity"
  input: "clip_by_global_norm/mul_229"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_230"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_229"
  op: "Identity"
  input: "clip_by_global_norm/mul_230"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_231"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_230"
  op: "Identity"
  input: "clip_by_global_norm/mul_231"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_232"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_231"
  op: "Identity"
  input: "clip_by_global_norm/mul_232"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_233"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_232"
  op: "Identity"
  input: "clip_by_global_norm/mul_233"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_234"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_233"
  op: "Identity"
  input: "clip_by_global_norm/mul_234"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_235"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_234"
  op: "Identity"
  input: "clip_by_global_norm/mul_235"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_236"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_235"
  op: "Identity"
  input: "clip_by_global_norm/mul_236"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_237"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_236"
  op: "Identity"
  input: "clip_by_global_norm/mul_237"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_238"
  op: "Mul"
  input: "gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_237"
  op: "Identity"
  input: "clip_by_global_norm/mul_238"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_239"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_238"
  op: "Identity"
  input: "clip_by_global_norm/mul_239"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_240"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_239"
  op: "Identity"
  input: "clip_by_global_norm/mul_240"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_241"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_240"
  op: "Identity"
  input: "clip_by_global_norm/mul_241"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_242"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_241"
  op: "Identity"
  input: "clip_by_global_norm/mul_242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_243"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_242"
  op: "Identity"
  input: "clip_by_global_norm/mul_243"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_244"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_243"
  op: "Identity"
  input: "clip_by_global_norm/mul_244"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_245"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_244"
  op: "Identity"
  input: "clip_by_global_norm/mul_245"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_246"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_245"
  op: "Identity"
  input: "clip_by_global_norm/mul_246"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_247"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_246"
  op: "Identity"
  input: "clip_by_global_norm/mul_247"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_248"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_247"
  op: "Identity"
  input: "clip_by_global_norm/mul_248"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_249"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_248"
  op: "Identity"
  input: "clip_by_global_norm/mul_249"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_250"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_249"
  op: "Identity"
  input: "clip_by_global_norm/mul_250"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_251"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_250"
  op: "Identity"
  input: "clip_by_global_norm/mul_251"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_252"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_251"
  op: "Identity"
  input: "clip_by_global_norm/mul_252"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_253"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_252"
  op: "Identity"
  input: "clip_by_global_norm/mul_253"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_254"
  op: "Mul"
  input: "gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_253"
  op: "Identity"
  input: "clip_by_global_norm/mul_254"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_255"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_254"
  op: "Identity"
  input: "clip_by_global_norm/mul_255"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_256"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_255"
  op: "Identity"
  input: "clip_by_global_norm/mul_256"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_257"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_256"
  op: "Identity"
  input: "clip_by_global_norm/mul_257"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_258"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_257"
  op: "Identity"
  input: "clip_by_global_norm/mul_258"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_259"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_258"
  op: "Identity"
  input: "clip_by_global_norm/mul_259"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_260"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_259"
  op: "Identity"
  input: "clip_by_global_norm/mul_260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_261"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_260"
  op: "Identity"
  input: "clip_by_global_norm/mul_261"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_262"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_261"
  op: "Identity"
  input: "clip_by_global_norm/mul_262"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_263"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_262"
  op: "Identity"
  input: "clip_by_global_norm/mul_263"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_264"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_263"
  op: "Identity"
  input: "clip_by_global_norm/mul_264"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_265"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_264"
  op: "Identity"
  input: "clip_by_global_norm/mul_265"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_266"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_265"
  op: "Identity"
  input: "clip_by_global_norm/mul_266"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_267"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_266"
  op: "Identity"
  input: "clip_by_global_norm/mul_267"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_268"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_267"
  op: "Identity"
  input: "clip_by_global_norm/mul_268"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_269"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_268"
  op: "Identity"
  input: "clip_by_global_norm/mul_269"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_270"
  op: "Mul"
  input: "gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_269"
  op: "Identity"
  input: "clip_by_global_norm/mul_270"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_271"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_270"
  op: "Identity"
  input: "clip_by_global_norm/mul_271"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_272"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_271"
  op: "Identity"
  input: "clip_by_global_norm/mul_272"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_273"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_272"
  op: "Identity"
  input: "clip_by_global_norm/mul_273"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_274"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_273"
  op: "Identity"
  input: "clip_by_global_norm/mul_274"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_275"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_274"
  op: "Identity"
  input: "clip_by_global_norm/mul_275"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_276"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_275"
  op: "Identity"
  input: "clip_by_global_norm/mul_276"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_277"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_276"
  op: "Identity"
  input: "clip_by_global_norm/mul_277"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_278"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_277"
  op: "Identity"
  input: "clip_by_global_norm/mul_278"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_279"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_278"
  op: "Identity"
  input: "clip_by_global_norm/mul_279"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_280"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_279"
  op: "Identity"
  input: "clip_by_global_norm/mul_280"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_281"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_280"
  op: "Identity"
  input: "clip_by_global_norm/mul_281"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_282"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_281"
  op: "Identity"
  input: "clip_by_global_norm/mul_282"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_283"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_282"
  op: "Identity"
  input: "clip_by_global_norm/mul_283"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_284"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_283"
  op: "Identity"
  input: "clip_by_global_norm/mul_284"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_285"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_284"
  op: "Identity"
  input: "clip_by_global_norm/mul_285"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_286"
  op: "Mul"
  input: "gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_285"
  op: "Identity"
  input: "clip_by_global_norm/mul_286"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_287"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_286"
  op: "Identity"
  input: "clip_by_global_norm/mul_287"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_288"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_287"
  op: "Identity"
  input: "clip_by_global_norm/mul_288"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_289"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_288"
  op: "Identity"
  input: "clip_by_global_norm/mul_289"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_290"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_289"
  op: "Identity"
  input: "clip_by_global_norm/mul_290"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_291"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_290"
  op: "Identity"
  input: "clip_by_global_norm/mul_291"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_292"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_291"
  op: "Identity"
  input: "clip_by_global_norm/mul_292"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_293"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_292"
  op: "Identity"
  input: "clip_by_global_norm/mul_293"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_294"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_293"
  op: "Identity"
  input: "clip_by_global_norm/mul_294"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_295"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_294"
  op: "Identity"
  input: "clip_by_global_norm/mul_295"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_296"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_295"
  op: "Identity"
  input: "clip_by_global_norm/mul_296"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_297"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_296"
  op: "Identity"
  input: "clip_by_global_norm/mul_297"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_298"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_297"
  op: "Identity"
  input: "clip_by_global_norm/mul_298"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_299"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_298"
  op: "Identity"
  input: "clip_by_global_norm/mul_299"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_300"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_299"
  op: "Identity"
  input: "clip_by_global_norm/mul_300"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_301"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_300"
  op: "Identity"
  input: "clip_by_global_norm/mul_301"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_302"
  op: "Mul"
  input: "gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_301"
  op: "Identity"
  input: "clip_by_global_norm/mul_302"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_303"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_302"
  op: "Identity"
  input: "clip_by_global_norm/mul_303"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_304"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_303"
  op: "Identity"
  input: "clip_by_global_norm/mul_304"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_305"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_304"
  op: "Identity"
  input: "clip_by_global_norm/mul_305"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_306"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_305"
  op: "Identity"
  input: "clip_by_global_norm/mul_306"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_307"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_306"
  op: "Identity"
  input: "clip_by_global_norm/mul_307"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_308"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_307"
  op: "Identity"
  input: "clip_by_global_norm/mul_308"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_309"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_308"
  op: "Identity"
  input: "clip_by_global_norm/mul_309"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_310"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_309"
  op: "Identity"
  input: "clip_by_global_norm/mul_310"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_311"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_310"
  op: "Identity"
  input: "clip_by_global_norm/mul_311"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_312"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_311"
  op: "Identity"
  input: "clip_by_global_norm/mul_312"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_313"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_312"
  op: "Identity"
  input: "clip_by_global_norm/mul_313"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_314"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_313"
  op: "Identity"
  input: "clip_by_global_norm/mul_314"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_315"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_314"
  op: "Identity"
  input: "clip_by_global_norm/mul_315"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_316"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_315"
  op: "Identity"
  input: "clip_by_global_norm/mul_316"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_317"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_316"
  op: "Identity"
  input: "clip_by_global_norm/mul_317"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_318"
  op: "Mul"
  input: "gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_317"
  op: "Identity"
  input: "clip_by_global_norm/mul_318"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_319"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_318"
  op: "Identity"
  input: "clip_by_global_norm/mul_319"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_320"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_319"
  op: "Identity"
  input: "clip_by_global_norm/mul_320"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_321"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_320"
  op: "Identity"
  input: "clip_by_global_norm/mul_321"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_322"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_321"
  op: "Identity"
  input: "clip_by_global_norm/mul_322"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_323"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_322"
  op: "Identity"
  input: "clip_by_global_norm/mul_323"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_324"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_323"
  op: "Identity"
  input: "clip_by_global_norm/mul_324"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_325"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_324"
  op: "Identity"
  input: "clip_by_global_norm/mul_325"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_326"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_325"
  op: "Identity"
  input: "clip_by_global_norm/mul_326"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_327"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_326"
  op: "Identity"
  input: "clip_by_global_norm/mul_327"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_328"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_327"
  op: "Identity"
  input: "clip_by_global_norm/mul_328"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_329"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_328"
  op: "Identity"
  input: "clip_by_global_norm/mul_329"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_330"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_329"
  op: "Identity"
  input: "clip_by_global_norm/mul_330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_331"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_330"
  op: "Identity"
  input: "clip_by_global_norm/mul_331"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_332"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_331"
  op: "Identity"
  input: "clip_by_global_norm/mul_332"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_333"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_332"
  op: "Identity"
  input: "clip_by_global_norm/mul_333"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_334"
  op: "Mul"
  input: "gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_333"
  op: "Identity"
  input: "clip_by_global_norm/mul_334"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_335"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_334"
  op: "Identity"
  input: "clip_by_global_norm/mul_335"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_336"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_335"
  op: "Identity"
  input: "clip_by_global_norm/mul_336"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_337"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_336"
  op: "Identity"
  input: "clip_by_global_norm/mul_337"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_338"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_337"
  op: "Identity"
  input: "clip_by_global_norm/mul_338"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_339"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_338"
  op: "Identity"
  input: "clip_by_global_norm/mul_339"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_340"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_339"
  op: "Identity"
  input: "clip_by_global_norm/mul_340"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_341"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_340"
  op: "Identity"
  input: "clip_by_global_norm/mul_341"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_342"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_341"
  op: "Identity"
  input: "clip_by_global_norm/mul_342"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_343"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_342"
  op: "Identity"
  input: "clip_by_global_norm/mul_343"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_344"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_343"
  op: "Identity"
  input: "clip_by_global_norm/mul_344"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_345"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_344"
  op: "Identity"
  input: "clip_by_global_norm/mul_345"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_346"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_345"
  op: "Identity"
  input: "clip_by_global_norm/mul_346"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_347"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_346"
  op: "Identity"
  input: "clip_by_global_norm/mul_347"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_348"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_347"
  op: "Identity"
  input: "clip_by_global_norm/mul_348"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_349"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_348"
  op: "Identity"
  input: "clip_by_global_norm/mul_349"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_350"
  op: "Mul"
  input: "gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_349"
  op: "Identity"
  input: "clip_by_global_norm/mul_350"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_351"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_350"
  op: "Identity"
  input: "clip_by_global_norm/mul_351"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_352"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_351"
  op: "Identity"
  input: "clip_by_global_norm/mul_352"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_353"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_352"
  op: "Identity"
  input: "clip_by_global_norm/mul_353"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_354"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_353"
  op: "Identity"
  input: "clip_by_global_norm/mul_354"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_355"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_354"
  op: "Identity"
  input: "clip_by_global_norm/mul_355"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_356"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_355"
  op: "Identity"
  input: "clip_by_global_norm/mul_356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_357"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_356"
  op: "Identity"
  input: "clip_by_global_norm/mul_357"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_358"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_357"
  op: "Identity"
  input: "clip_by_global_norm/mul_358"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_359"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_358"
  op: "Identity"
  input: "clip_by_global_norm/mul_359"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_360"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_359"
  op: "Identity"
  input: "clip_by_global_norm/mul_360"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_361"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_360"
  op: "Identity"
  input: "clip_by_global_norm/mul_361"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_362"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_361"
  op: "Identity"
  input: "clip_by_global_norm/mul_362"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_363"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_362"
  op: "Identity"
  input: "clip_by_global_norm/mul_363"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_364"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_363"
  op: "Identity"
  input: "clip_by_global_norm/mul_364"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_365"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_364"
  op: "Identity"
  input: "clip_by_global_norm/mul_365"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_366"
  op: "Mul"
  input: "gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_365"
  op: "Identity"
  input: "clip_by_global_norm/mul_366"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_367"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_366"
  op: "Identity"
  input: "clip_by_global_norm/mul_367"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_368"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_367"
  op: "Identity"
  input: "clip_by_global_norm/mul_368"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_369"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_368"
  op: "Identity"
  input: "clip_by_global_norm/mul_369"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_370"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_369"
  op: "Identity"
  input: "clip_by_global_norm/mul_370"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_371"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_370"
  op: "Identity"
  input: "clip_by_global_norm/mul_371"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_372"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_371"
  op: "Identity"
  input: "clip_by_global_norm/mul_372"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_373"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_372"
  op: "Identity"
  input: "clip_by_global_norm/mul_373"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_374"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_373"
  op: "Identity"
  input: "clip_by_global_norm/mul_374"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_375"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_374"
  op: "Identity"
  input: "clip_by_global_norm/mul_375"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_376"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_375"
  op: "Identity"
  input: "clip_by_global_norm/mul_376"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_377"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_376"
  op: "Identity"
  input: "clip_by_global_norm/mul_377"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_378"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_377"
  op: "Identity"
  input: "clip_by_global_norm/mul_378"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_379"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_378"
  op: "Identity"
  input: "clip_by_global_norm/mul_379"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_380"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_379"
  op: "Identity"
  input: "clip_by_global_norm/mul_380"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_381"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_380"
  op: "Identity"
  input: "clip_by_global_norm/mul_381"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_382"
  op: "Mul"
  input: "gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_381"
  op: "Identity"
  input: "clip_by_global_norm/mul_382"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_383"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_382"
  op: "Identity"
  input: "clip_by_global_norm/mul_383"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_384"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_383"
  op: "Identity"
  input: "clip_by_global_norm/mul_384"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_385"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_384"
  op: "Identity"
  input: "clip_by_global_norm/mul_385"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_386"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_385"
  op: "Identity"
  input: "clip_by_global_norm/mul_386"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_387"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_386"
  op: "Identity"
  input: "clip_by_global_norm/mul_387"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_388"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_387"
  op: "Identity"
  input: "clip_by_global_norm/mul_388"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_389"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_388"
  op: "Identity"
  input: "clip_by_global_norm/mul_389"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_390"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_389"
  op: "Identity"
  input: "clip_by_global_norm/mul_390"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_391"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_390"
  op: "Identity"
  input: "clip_by_global_norm/mul_391"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_392"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_391"
  op: "Identity"
  input: "clip_by_global_norm/mul_392"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_393"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_392"
  op: "Identity"
  input: "clip_by_global_norm/mul_393"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_394"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_393"
  op: "Identity"
  input: "clip_by_global_norm/mul_394"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_395"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_394"
  op: "Identity"
  input: "clip_by_global_norm/mul_395"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_396"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_395"
  op: "Identity"
  input: "clip_by_global_norm/mul_396"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_397"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_396"
  op: "Identity"
  input: "clip_by_global_norm/mul_397"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_398"
  op: "Mul"
  input: "gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_397"
  op: "Identity"
  input: "clip_by_global_norm/mul_398"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_399"
  op: "Mul"
  input: "gradients/discriminator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_398"
  op: "Identity"
  input: "clip_by_global_norm/mul_399"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_400"
  op: "Mul"
  input: "gradients/discriminator_predictions/dense/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_399"
  op: "Identity"
  input: "clip_by_global_norm/mul_400"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_401"
  op: "Mul"
  input: "gradients/discriminator_predictions/dense_1/Tensordot/MatMul_grad/MatMul_1"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense_1/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_400"
  op: "Identity"
  input: "clip_by_global_norm/mul_401"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense_1/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/mul_402"
  op: "Mul"
  input: "gradients/discriminator_predictions/dense_1/BiasAdd_grad/BiasAddGrad"
  input: "clip_by_global_norm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense_1/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "clip_by_global_norm/clip_by_global_norm/_401"
  op: "Identity"
  input: "clip_by_global_norm/mul_402"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense_1/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/MatMul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_0"
  input: "CrossReplicaSum/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/MatMul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_1/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_1"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_1"
  input: "CrossReplicaSum_1/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_2/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/Slice_grad/XlaDynamicUpdateSlice"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_2"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_2"
  input: "CrossReplicaSum_2/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/Slice_grad/XlaDynamicUpdateSlice"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_3/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_3"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_3"
  input: "CrossReplicaSum_3/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_4/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_4"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_4"
  input: "CrossReplicaSum_4/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_1/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_5/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/embeddings_project/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_5"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_5"
  input: "CrossReplicaSum_5/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/embeddings_project/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_6/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/embeddings_project/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_6"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_6"
  input: "CrossReplicaSum_6/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/embeddings_project/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_7/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_7"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_7"
  input: "CrossReplicaSum_7/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_8/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_8"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_8"
  input: "CrossReplicaSum_8/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_9/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_9"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_9"
  input: "CrossReplicaSum_9/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_10/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_10"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_10"
  input: "CrossReplicaSum_10/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_11/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_11"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_11"
  input: "CrossReplicaSum_11/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_12/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_12"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_12"
  input: "CrossReplicaSum_12/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_13/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_13"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_13"
  input: "CrossReplicaSum_13/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_14/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_14"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_14"
  input: "CrossReplicaSum_14/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_15/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_15"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_15"
  input: "CrossReplicaSum_15/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_16/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_16"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_16"
  input: "CrossReplicaSum_16/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_17/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_17"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_17"
  input: "CrossReplicaSum_17/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_18/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_18"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_18"
  input: "CrossReplicaSum_18/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_19/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_19"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_19"
  input: "CrossReplicaSum_19/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_20/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_20"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_20"
  input: "CrossReplicaSum_20/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_21/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_21"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_21"
  input: "CrossReplicaSum_21/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_22/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_22"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_22"
  input: "CrossReplicaSum_22/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_23/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_23"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_23"
  input: "CrossReplicaSum_23/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_24/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_24"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_24"
  input: "CrossReplicaSum_24/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_25/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_25"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_25"
  input: "CrossReplicaSum_25/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_26/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_26"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_26"
  input: "CrossReplicaSum_26/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_27/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_27"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_27"
  input: "CrossReplicaSum_27/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_28/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_28"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_28"
  input: "CrossReplicaSum_28/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_29/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_29"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_29"
  input: "CrossReplicaSum_29/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_30/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_30"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_30"
  input: "CrossReplicaSum_30/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_31/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_31"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_31"
  input: "CrossReplicaSum_31/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_32/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_32"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_32"
  input: "CrossReplicaSum_32/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_33/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_33"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_33"
  input: "CrossReplicaSum_33/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_34/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_34"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_34"
  input: "CrossReplicaSum_34/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_35/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_35"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_35"
  input: "CrossReplicaSum_35/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_36/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_36"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_36"
  input: "CrossReplicaSum_36/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_37/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_37"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_37"
  input: "CrossReplicaSum_37/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_38/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_38"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_38"
  input: "CrossReplicaSum_38/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_39/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_39"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_39"
  input: "CrossReplicaSum_39/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_40/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_40"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_40"
  input: "CrossReplicaSum_40/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_41/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_41"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_41"
  input: "CrossReplicaSum_41/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_42/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_42"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_42"
  input: "CrossReplicaSum_42/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_43/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_43"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_43"
  input: "CrossReplicaSum_43/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_44/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_44"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_44"
  input: "CrossReplicaSum_44/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_45/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_45"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_45"
  input: "CrossReplicaSum_45/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_46/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_46"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_46"
  input: "CrossReplicaSum_46/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_47/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_47"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_47"
  input: "CrossReplicaSum_47/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_48/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_48"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_48"
  input: "CrossReplicaSum_48/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_49/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_49"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_49"
  input: "CrossReplicaSum_49/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_50/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_50"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_50"
  input: "CrossReplicaSum_50/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_51/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_51"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_51"
  input: "CrossReplicaSum_51/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_52/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_52"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_52"
  input: "CrossReplicaSum_52/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_53/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_53"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_53"
  input: "CrossReplicaSum_53/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_54/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_54"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_54"
  input: "CrossReplicaSum_54/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_55/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_55"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_55"
  input: "CrossReplicaSum_55/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_56/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_56"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_56"
  input: "CrossReplicaSum_56/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_57/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_57"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_57"
  input: "CrossReplicaSum_57/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_58/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_58"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_58"
  input: "CrossReplicaSum_58/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_59/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_59"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_59"
  input: "CrossReplicaSum_59/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_60/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_60"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_60"
  input: "CrossReplicaSum_60/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_61/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_61"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_61"
  input: "CrossReplicaSum_61/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_62/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_62"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_62"
  input: "CrossReplicaSum_62/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_63/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_63"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_63"
  input: "CrossReplicaSum_63/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_64/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_64"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_64"
  input: "CrossReplicaSum_64/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_65/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_65"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_65"
  input: "CrossReplicaSum_65/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_66/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_66"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_66"
  input: "CrossReplicaSum_66/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_67/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_67"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_67"
  input: "CrossReplicaSum_67/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_68/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_68"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_68"
  input: "CrossReplicaSum_68/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_69/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_69"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_69"
  input: "CrossReplicaSum_69/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_70/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_70"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_70"
  input: "CrossReplicaSum_70/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_71/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_71"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_71"
  input: "CrossReplicaSum_71/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_72/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_72"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_72"
  input: "CrossReplicaSum_72/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_73/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_73"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_73"
  input: "CrossReplicaSum_73/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_74/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_74"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_74"
  input: "CrossReplicaSum_74/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_75/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_75"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_75"
  input: "CrossReplicaSum_75/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_76/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_76"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_76"
  input: "CrossReplicaSum_76/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_77/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_77"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_77"
  input: "CrossReplicaSum_77/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_78/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_78"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_78"
  input: "CrossReplicaSum_78/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_79/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_79"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_79"
  input: "CrossReplicaSum_79/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_80/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_80"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_80"
  input: "CrossReplicaSum_80/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_81/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_81"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_81"
  input: "CrossReplicaSum_81/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_82/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_82"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_82"
  input: "CrossReplicaSum_82/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_83/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_83"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_83"
  input: "CrossReplicaSum_83/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_84/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_84"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_84"
  input: "CrossReplicaSum_84/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_85/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_85"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_85"
  input: "CrossReplicaSum_85/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_86/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_86"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_86"
  input: "CrossReplicaSum_86/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_87/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_87"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_87"
  input: "CrossReplicaSum_87/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_88/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_88"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_88"
  input: "CrossReplicaSum_88/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_89/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_89"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_89"
  input: "CrossReplicaSum_89/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_90/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_90"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_90"
  input: "CrossReplicaSum_90/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_91/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_91"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_91"
  input: "CrossReplicaSum_91/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_92/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_92"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_92"
  input: "CrossReplicaSum_92/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_93/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_93"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_93"
  input: "CrossReplicaSum_93/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_94/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_94"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_94"
  input: "CrossReplicaSum_94/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_95/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_95"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_95"
  input: "CrossReplicaSum_95/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_96/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_96"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_96"
  input: "CrossReplicaSum_96/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_97/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_97"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_97"
  input: "CrossReplicaSum_97/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_98/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_98"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_98"
  input: "CrossReplicaSum_98/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_99/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_99"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_99"
  input: "CrossReplicaSum_99/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_100/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_100"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_100"
  input: "CrossReplicaSum_100/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_101/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_101"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_101"
  input: "CrossReplicaSum_101/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_102/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_102"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_102"
  input: "CrossReplicaSum_102/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_103/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_103"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_103"
  input: "CrossReplicaSum_103/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_104/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_104"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_104"
  input: "CrossReplicaSum_104/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_105/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_105"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_105"
  input: "CrossReplicaSum_105/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_106/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_106"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_106"
  input: "CrossReplicaSum_106/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_107/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_107"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_107"
  input: "CrossReplicaSum_107/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_108/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_108"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_108"
  input: "CrossReplicaSum_108/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_109/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_109"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_109"
  input: "CrossReplicaSum_109/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_110/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_110"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_110"
  input: "CrossReplicaSum_110/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_111/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_111"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_111"
  input: "CrossReplicaSum_111/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_112/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_112"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_112"
  input: "CrossReplicaSum_112/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_113/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_113"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_113"
  input: "CrossReplicaSum_113/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_114/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_114"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_114"
  input: "CrossReplicaSum_114/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_115/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_115"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_115"
  input: "CrossReplicaSum_115/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_116/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_116"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_116"
  input: "CrossReplicaSum_116/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_117/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_117"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_117"
  input: "CrossReplicaSum_117/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_118/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_118"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_118"
  input: "CrossReplicaSum_118/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_119/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_119"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_119"
  input: "CrossReplicaSum_119/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_120/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_120"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_120"
  input: "CrossReplicaSum_120/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_121/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_121"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_121"
  input: "CrossReplicaSum_121/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_122/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_122"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_122"
  input: "CrossReplicaSum_122/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_123/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_123"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_123"
  input: "CrossReplicaSum_123/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_124/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_124"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_124"
  input: "CrossReplicaSum_124/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_125/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_125"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_125"
  input: "CrossReplicaSum_125/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_126/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_126"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_126"
  input: "CrossReplicaSum_126/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_127/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_127"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_127"
  input: "CrossReplicaSum_127/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_128/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_128"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_128"
  input: "CrossReplicaSum_128/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_129/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_129"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_129"
  input: "CrossReplicaSum_129/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_130/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_130"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_130"
  input: "CrossReplicaSum_130/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_131/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_131"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_131"
  input: "CrossReplicaSum_131/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_132/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_132"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_132"
  input: "CrossReplicaSum_132/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_133/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_133"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_133"
  input: "CrossReplicaSum_133/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_134/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_134"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_134"
  input: "CrossReplicaSum_134/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_135/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_135"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_135"
  input: "CrossReplicaSum_135/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_136/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_136"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_136"
  input: "CrossReplicaSum_136/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_137/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_137"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_137"
  input: "CrossReplicaSum_137/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_138/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_138"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_138"
  input: "CrossReplicaSum_138/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_139/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_139"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_139"
  input: "CrossReplicaSum_139/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_140/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_140"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_140"
  input: "CrossReplicaSum_140/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_141/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_141"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_141"
  input: "CrossReplicaSum_141/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_142/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_142"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_142"
  input: "CrossReplicaSum_142/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_143/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_143"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_143"
  input: "CrossReplicaSum_143/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_144/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_144"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_144"
  input: "CrossReplicaSum_144/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_145/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_145"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_145"
  input: "CrossReplicaSum_145/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_146/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_146"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_146"
  input: "CrossReplicaSum_146/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_147/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_147"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_147"
  input: "CrossReplicaSum_147/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_148/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_148"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_148"
  input: "CrossReplicaSum_148/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_149/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_149"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_149"
  input: "CrossReplicaSum_149/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_150/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_150"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_150"
  input: "CrossReplicaSum_150/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_151/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_151"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_151"
  input: "CrossReplicaSum_151/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_152/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_152"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_152"
  input: "CrossReplicaSum_152/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_153/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_153"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_153"
  input: "CrossReplicaSum_153/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_154/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_154"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_154"
  input: "CrossReplicaSum_154/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_155/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_155"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_155"
  input: "CrossReplicaSum_155/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_156/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_156"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_156"
  input: "CrossReplicaSum_156/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_157/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_157"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_157"
  input: "CrossReplicaSum_157/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_158/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_158"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_158"
  input: "CrossReplicaSum_158/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_159/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_159"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_159"
  input: "CrossReplicaSum_159/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_160/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_160"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_160"
  input: "CrossReplicaSum_160/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_161/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_161"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_161"
  input: "CrossReplicaSum_161/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_162/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_162"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_162"
  input: "CrossReplicaSum_162/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_163/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_163"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_163"
  input: "CrossReplicaSum_163/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_164/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_164"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_164"
  input: "CrossReplicaSum_164/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_165/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_165"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_165"
  input: "CrossReplicaSum_165/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_166/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_166"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_166"
  input: "CrossReplicaSum_166/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_167/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_167"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_167"
  input: "CrossReplicaSum_167/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_168/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_168"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_168"
  input: "CrossReplicaSum_168/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_169/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_169"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_169"
  input: "CrossReplicaSum_169/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_170/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_170"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_170"
  input: "CrossReplicaSum_170/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_171/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_171"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_171"
  input: "CrossReplicaSum_171/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_172/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_172"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_172"
  input: "CrossReplicaSum_172/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_173/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_173"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_173"
  input: "CrossReplicaSum_173/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_174/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_174"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_174"
  input: "CrossReplicaSum_174/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_175/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_175"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_175"
  input: "CrossReplicaSum_175/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_176/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_176"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_176"
  input: "CrossReplicaSum_176/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_177/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_177"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_177"
  input: "CrossReplicaSum_177/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_178/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_178"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_178"
  input: "CrossReplicaSum_178/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_179/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_179"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_179"
  input: "CrossReplicaSum_179/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_180/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_180"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_180"
  input: "CrossReplicaSum_180/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_181/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_181"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_181"
  input: "CrossReplicaSum_181/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_182/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_182"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_182"
  input: "CrossReplicaSum_182/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_183/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_183"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_183"
  input: "CrossReplicaSum_183/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_184/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_184"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_184"
  input: "CrossReplicaSum_184/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_185/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_185"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_185"
  input: "CrossReplicaSum_185/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_186/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_186"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_186"
  input: "CrossReplicaSum_186/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_187/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_187"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_187"
  input: "CrossReplicaSum_187/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_188/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_188"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_188"
  input: "CrossReplicaSum_188/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_189/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_189"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_189"
  input: "CrossReplicaSum_189/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_190/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_190"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_190"
  input: "CrossReplicaSum_190/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_191/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_191"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_191"
  input: "CrossReplicaSum_191/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_192/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_192"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_192"
  input: "CrossReplicaSum_192/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_193/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_193"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_193"
  input: "CrossReplicaSum_193/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_194/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_194"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_194"
  input: "CrossReplicaSum_194/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_195/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_195"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_195"
  input: "CrossReplicaSum_195/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_196/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_196"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_196"
  input: "CrossReplicaSum_196/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_197/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_197"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_197"
  input: "CrossReplicaSum_197/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_198/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_198"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_198"
  input: "CrossReplicaSum_198/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_199/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_199"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_199"
  input: "CrossReplicaSum_199/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_200/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_200"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_200"
  input: "CrossReplicaSum_200/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_201/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_201"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_201"
  input: "CrossReplicaSum_201/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_202/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_202"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_202"
  input: "CrossReplicaSum_202/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_203/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_203"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_203"
  input: "CrossReplicaSum_203/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/generator_predictions/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_204/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_204"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_204"
  input: "CrossReplicaSum_204/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_205/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_205"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_205"
  input: "CrossReplicaSum_205/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/embeddings_3/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_206/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_206"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_206"
  input: "CrossReplicaSum_206/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_207/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_207"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_207"
  input: "CrossReplicaSum_207/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_208/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_208"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_208"
  input: "CrossReplicaSum_208/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_209/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_209"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_209"
  input: "CrossReplicaSum_209/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_210/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_210"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_210"
  input: "CrossReplicaSum_210/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_211/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_211"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_211"
  input: "CrossReplicaSum_211/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_212/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_212"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_212"
  input: "CrossReplicaSum_212/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_213/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_213"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_213"
  input: "CrossReplicaSum_213/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_214/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_214"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_214"
  input: "CrossReplicaSum_214/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_215/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_215"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_215"
  input: "CrossReplicaSum_215/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_216/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_216"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_216"
  input: "CrossReplicaSum_216/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_217/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_217"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_217"
  input: "CrossReplicaSum_217/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_218/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_218"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_218"
  input: "CrossReplicaSum_218/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_219/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_219"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_219"
  input: "CrossReplicaSum_219/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_220/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_220"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_220"
  input: "CrossReplicaSum_220/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_221/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_221"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_221"
  input: "CrossReplicaSum_221/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_0/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_222/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_222"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_222"
  input: "CrossReplicaSum_222/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_223/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_223"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_223"
  input: "CrossReplicaSum_223/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_224/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_224"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_224"
  input: "CrossReplicaSum_224/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_225/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_225"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_225"
  input: "CrossReplicaSum_225/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_226/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_226"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_226"
  input: "CrossReplicaSum_226/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_227/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_227"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_227"
  input: "CrossReplicaSum_227/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_228/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_228"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_228"
  input: "CrossReplicaSum_228/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_229/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_229"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_229"
  input: "CrossReplicaSum_229/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_230/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_230"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_230"
  input: "CrossReplicaSum_230/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_231/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_231"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_231"
  input: "CrossReplicaSum_231/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_232/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_232"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_232"
  input: "CrossReplicaSum_232/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_233/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_233"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_233"
  input: "CrossReplicaSum_233/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_234/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_234"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_234"
  input: "CrossReplicaSum_234/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_235/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_235"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_235"
  input: "CrossReplicaSum_235/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_236/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_236"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_236"
  input: "CrossReplicaSum_236/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_237/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_237"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_237"
  input: "CrossReplicaSum_237/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_1/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_238/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_238"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_238"
  input: "CrossReplicaSum_238/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_239/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_239"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_239"
  input: "CrossReplicaSum_239/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_240/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_240"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_240"
  input: "CrossReplicaSum_240/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_241/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_241"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_241"
  input: "CrossReplicaSum_241/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_242/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_242"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_242"
  input: "CrossReplicaSum_242/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_243/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_243"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_243"
  input: "CrossReplicaSum_243/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_244/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_244"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_244"
  input: "CrossReplicaSum_244/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_245/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_245"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_245"
  input: "CrossReplicaSum_245/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_246/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_246"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_246"
  input: "CrossReplicaSum_246/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_247/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_247"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_247"
  input: "CrossReplicaSum_247/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_248/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_248"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_248"
  input: "CrossReplicaSum_248/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_249/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_249"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_249"
  input: "CrossReplicaSum_249/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_250/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_250"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_250"
  input: "CrossReplicaSum_250/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_251/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_251"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_251"
  input: "CrossReplicaSum_251/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_252/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_252"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_252"
  input: "CrossReplicaSum_252/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_253/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_253"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_253"
  input: "CrossReplicaSum_253/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_2/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_254/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_254"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_254"
  input: "CrossReplicaSum_254/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_255/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_255"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_255"
  input: "CrossReplicaSum_255/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_256/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_256"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_256"
  input: "CrossReplicaSum_256/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_257/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_257"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_257"
  input: "CrossReplicaSum_257/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_258/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_258"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_258"
  input: "CrossReplicaSum_258/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_259/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_259"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_259"
  input: "CrossReplicaSum_259/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_260/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_260"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_260"
  input: "CrossReplicaSum_260/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_261/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_261"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_261"
  input: "CrossReplicaSum_261/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_262/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_262"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_262"
  input: "CrossReplicaSum_262/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_263/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_263"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_263"
  input: "CrossReplicaSum_263/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_264/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_264"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_264"
  input: "CrossReplicaSum_264/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_265/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_265"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_265"
  input: "CrossReplicaSum_265/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_266/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_266"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_266"
  input: "CrossReplicaSum_266/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_267/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_267"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_267"
  input: "CrossReplicaSum_267/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_268/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_268"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_268"
  input: "CrossReplicaSum_268/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_269/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_269"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_269"
  input: "CrossReplicaSum_269/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_3/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_270/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_270"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_270"
  input: "CrossReplicaSum_270/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_271/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_271"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_271"
  input: "CrossReplicaSum_271/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_272/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_272"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_272"
  input: "CrossReplicaSum_272/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_273/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_273"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_273"
  input: "CrossReplicaSum_273/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_274/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_274"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_274"
  input: "CrossReplicaSum_274/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_275/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_275"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_275"
  input: "CrossReplicaSum_275/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_276/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_276"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_276"
  input: "CrossReplicaSum_276/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_277/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_277"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_277"
  input: "CrossReplicaSum_277/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_278/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_278"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_278"
  input: "CrossReplicaSum_278/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_279/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_279"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_279"
  input: "CrossReplicaSum_279/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_280/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_280"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_280"
  input: "CrossReplicaSum_280/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_281/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_281"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_281"
  input: "CrossReplicaSum_281/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_282/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_282"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_282"
  input: "CrossReplicaSum_282/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_283/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_283"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_283"
  input: "CrossReplicaSum_283/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_284/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_284"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_284"
  input: "CrossReplicaSum_284/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_285/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_285"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_285"
  input: "CrossReplicaSum_285/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_4/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_286/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_286"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_286"
  input: "CrossReplicaSum_286/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_287/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_287"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_287"
  input: "CrossReplicaSum_287/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_288/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_288"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_288"
  input: "CrossReplicaSum_288/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_289/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_289"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_289"
  input: "CrossReplicaSum_289/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_290/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_290"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_290"
  input: "CrossReplicaSum_290/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_291/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_291"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_291"
  input: "CrossReplicaSum_291/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_292/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_292"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_292"
  input: "CrossReplicaSum_292/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_293/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_293"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_293"
  input: "CrossReplicaSum_293/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_294/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_294"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_294"
  input: "CrossReplicaSum_294/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_295/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_295"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_295"
  input: "CrossReplicaSum_295/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_296/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_296"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_296"
  input: "CrossReplicaSum_296/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_297/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_297"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_297"
  input: "CrossReplicaSum_297/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_298/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_298"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_298"
  input: "CrossReplicaSum_298/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_299/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_299"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_299"
  input: "CrossReplicaSum_299/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_300/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_300"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_300"
  input: "CrossReplicaSum_300/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_301/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_301"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_301"
  input: "CrossReplicaSum_301/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_5/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_302/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_302"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_302"
  input: "CrossReplicaSum_302/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_303/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_303"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_303"
  input: "CrossReplicaSum_303/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_304/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_304"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_304"
  input: "CrossReplicaSum_304/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_305/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_305"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_305"
  input: "CrossReplicaSum_305/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_306/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_306"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_306"
  input: "CrossReplicaSum_306/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_307/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_307"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_307"
  input: "CrossReplicaSum_307/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_308/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_308"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_308"
  input: "CrossReplicaSum_308/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_309/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_309"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_309"
  input: "CrossReplicaSum_309/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_310/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_310"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_310"
  input: "CrossReplicaSum_310/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_311/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_311"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_311"
  input: "CrossReplicaSum_311/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_312/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_312"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_312"
  input: "CrossReplicaSum_312/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_313/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_313"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_313"
  input: "CrossReplicaSum_313/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_314/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_314"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_314"
  input: "CrossReplicaSum_314/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_315/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_315"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_315"
  input: "CrossReplicaSum_315/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_316/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_316"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_316"
  input: "CrossReplicaSum_316/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_317/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_317"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_317"
  input: "CrossReplicaSum_317/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_6/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_318/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_318"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_318"
  input: "CrossReplicaSum_318/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_319/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_319"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_319"
  input: "CrossReplicaSum_319/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_320/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_320"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_320"
  input: "CrossReplicaSum_320/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_321/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_321"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_321"
  input: "CrossReplicaSum_321/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_322/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_322"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_322"
  input: "CrossReplicaSum_322/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_323/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_323"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_323"
  input: "CrossReplicaSum_323/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_324/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_324"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_324"
  input: "CrossReplicaSum_324/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_325/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_325"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_325"
  input: "CrossReplicaSum_325/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_326/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_326"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_326"
  input: "CrossReplicaSum_326/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_327/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_327"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_327"
  input: "CrossReplicaSum_327/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_328/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_328"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_328"
  input: "CrossReplicaSum_328/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_329/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_329"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_329"
  input: "CrossReplicaSum_329/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_330/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_330"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_330"
  input: "CrossReplicaSum_330/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_331/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_331"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_331"
  input: "CrossReplicaSum_331/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_332/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_332"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_332"
  input: "CrossReplicaSum_332/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_333/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_333"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_333"
  input: "CrossReplicaSum_333/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_7/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_334/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_334"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_334"
  input: "CrossReplicaSum_334/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_335/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_335"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_335"
  input: "CrossReplicaSum_335/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_336/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_336"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_336"
  input: "CrossReplicaSum_336/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_337/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_337"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_337"
  input: "CrossReplicaSum_337/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_338/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_338"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_338"
  input: "CrossReplicaSum_338/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_339/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_339"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_339"
  input: "CrossReplicaSum_339/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_340/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_340"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_340"
  input: "CrossReplicaSum_340/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_341/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_341"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_341"
  input: "CrossReplicaSum_341/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_342/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_342"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_342"
  input: "CrossReplicaSum_342/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_343/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_343"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_343"
  input: "CrossReplicaSum_343/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_344/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_344"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_344"
  input: "CrossReplicaSum_344/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_345/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_345"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_345"
  input: "CrossReplicaSum_345/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_346/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_346"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_346"
  input: "CrossReplicaSum_346/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_347/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_347"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_347"
  input: "CrossReplicaSum_347/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_348/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_348"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_348"
  input: "CrossReplicaSum_348/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_349/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_349"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_349"
  input: "CrossReplicaSum_349/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_8/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_350/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_350"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_350"
  input: "CrossReplicaSum_350/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_351/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_351"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_351"
  input: "CrossReplicaSum_351/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_352/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_352"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_352"
  input: "CrossReplicaSum_352/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_353/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_353"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_353"
  input: "CrossReplicaSum_353/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_354/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_354"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_354"
  input: "CrossReplicaSum_354/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_355/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_355"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_355"
  input: "CrossReplicaSum_355/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_356/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_356"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_356"
  input: "CrossReplicaSum_356/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_357/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_357"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_357"
  input: "CrossReplicaSum_357/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_358/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_358"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_358"
  input: "CrossReplicaSum_358/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_359/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_359"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_359"
  input: "CrossReplicaSum_359/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_360/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_360"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_360"
  input: "CrossReplicaSum_360/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_361/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_361"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_361"
  input: "CrossReplicaSum_361/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_362/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_362"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_362"
  input: "CrossReplicaSum_362/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_363/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_363"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_363"
  input: "CrossReplicaSum_363/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_364/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_364"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_364"
  input: "CrossReplicaSum_364/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_365/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_365"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_365"
  input: "CrossReplicaSum_365/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_9/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_366/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_366"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_366"
  input: "CrossReplicaSum_366/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_367/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_367"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_367"
  input: "CrossReplicaSum_367/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_368/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_368"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_368"
  input: "CrossReplicaSum_368/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_369/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_369"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_369"
  input: "CrossReplicaSum_369/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_370/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_370"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_370"
  input: "CrossReplicaSum_370/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_371/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_371"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_371"
  input: "CrossReplicaSum_371/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_372/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_372"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_372"
  input: "CrossReplicaSum_372/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_373/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_373"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_373"
  input: "CrossReplicaSum_373/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_374/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_374"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_374"
  input: "CrossReplicaSum_374/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_375/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_375"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_375"
  input: "CrossReplicaSum_375/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_376/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_376"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_376"
  input: "CrossReplicaSum_376/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_377/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_377"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_377"
  input: "CrossReplicaSum_377/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_378/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_378"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_378"
  input: "CrossReplicaSum_378/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_379/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_379"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_379"
  input: "CrossReplicaSum_379/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_380/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_380"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_380"
  input: "CrossReplicaSum_380/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_381/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_381"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_381"
  input: "CrossReplicaSum_381/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_10/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_382/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_382"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_382"
  input: "CrossReplicaSum_382/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_383/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_383"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_383"
  input: "CrossReplicaSum_383/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_384/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_384"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_384"
  input: "CrossReplicaSum_384/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_385/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_385"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_385"
  input: "CrossReplicaSum_385/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_386/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_386"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_386"
  input: "CrossReplicaSum_386/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_387/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_387"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_387"
  input: "CrossReplicaSum_387/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_388/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_388"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_388"
  input: "CrossReplicaSum_388/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_389/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_389"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_389"
  input: "CrossReplicaSum_389/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_390/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_390"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_390"
  input: "CrossReplicaSum_390/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_391/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_391"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_391"
  input: "CrossReplicaSum_391/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_392/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_392"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_392"
  input: "CrossReplicaSum_392/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_393/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_393"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_393"
  input: "CrossReplicaSum_393/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_394/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_394"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_394"
  input: "CrossReplicaSum_394/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/dense/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_395/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_395"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_395"
  input: "CrossReplicaSum_395/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_396/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_396"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_396"
  input: "CrossReplicaSum_396/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/mul_grad/Reshape_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_397/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_397"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_397"
  input: "CrossReplicaSum_397/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/electra/encoder/layer_11/output/LayerNorm/batchnorm/sub_grad/Reshape"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_398/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_398"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_398"
  input: "CrossReplicaSum_398/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_399/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_399"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_399"
  input: "CrossReplicaSum_399/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_400/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense_1/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_400"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_400"
  input: "CrossReplicaSum_400/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense_1/Tensordot/MatMul_grad/MatMul_1"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "CrossReplicaSum_401/group_assignment"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense_1/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
          dim {
            size: 8
          }
        }
        tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000\004\000\000\000\005\000\000\000\006\000\000\000\007\000\000\000"
      }
    }
  }
}
node {
  name: "CrossReplicaSum_401"
  op: "CrossReplicaSum"
  input: "clip_by_global_norm/clip_by_global_norm/_401"
  input: "CrossReplicaSum_401/group_assignment"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@gradients/discriminator_predictions/dense_1/BiasAdd_grad/BiasAddGrad"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "Y\304\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/embeddings/word_embeddings/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/embeddings/word_embeddings/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 50265
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings/word_embeddings/adam_m"
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/word_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings/word_embeddings/adam_m"
  input: "electra/embeddings/word_embeddings/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/word_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "Y\304\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/embeddings/word_embeddings/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/embeddings/word_embeddings/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/word_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 50265
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings/word_embeddings/adam_v"
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/word_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings/word_embeddings/adam_v"
  input: "electra/embeddings/word_embeddings/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/word_embeddings/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/word_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_22/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_22/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_22/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_22/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings/word_embeddings/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_22"
  op: "Mul"
  input: "Mul_22/x"
  input: "Mul_22/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_23/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_23"
  op: "Mul"
  input: "Mul_23/x"
  input: "CrossReplicaSum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_12"
  op: "AddV2"
  input: "Mul_22"
  input: "Mul_23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_24/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_24/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_24/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_24/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings/word_embeddings/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_24"
  op: "Mul"
  input: "Mul_24/x"
  input: "Mul_24/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square"
  op: "Square"
  input: "CrossReplicaSum"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_25/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_25"
  op: "Mul"
  input: "Mul_25/x"
  input: "Square"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_13"
  op: "AddV2"
  input: "Mul_24"
  input: "Mul_25"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt"
  op: "Sqrt"
  input: "add_13"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_14/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_14"
  op: "AddV2"
  input: "Sqrt"
  input: "add_14/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_3"
  op: "RealDiv"
  input: "add_12"
  input: "add_14"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_26/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_26"
  op: "Mul"
  input: "mul_26/x"
  input: "ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_15"
  op: "AddV2"
  input: "truediv_3"
  input: "mul_26"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_27"
  op: "Mul"
  input: "mul_21"
  input: "add_15"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1"
  op: "ReadVariableOp"
  input: "electra/embeddings/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_4"
  op: "Sub"
  input: "ReadVariableOp_1"
  input: "mul_27"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp"
  op: "AssignVariableOp"
  input: "electra/embeddings/MatMul/ReadVariableOp/Enter"
  input: "sub_4"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_2"
  op: "ReadVariableOp"
  input: "electra/embeddings/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1"
  op: "AssignVariableOp"
  input: "Mul_22/ReadVariableOp/Enter"
  input: "add_12"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_3"
  op: "ReadVariableOp"
  input: "Mul_22/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_2"
  op: "AssignVariableOp"
  input: "Mul_24/ReadVariableOp/Enter"
  input: "add_13"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_4"
  op: "ReadVariableOp"
  input: "Mul_24/ReadVariableOp/Enter"
  input: "^AssignVariableOp_2"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\002\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/embeddings/token_type_embeddings/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/embeddings/token_type_embeddings/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 2
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings/token_type_embeddings/adam_m"
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/token_type_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings/token_type_embeddings/adam_m"
  input: "electra/embeddings/token_type_embeddings/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/token_type_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\002\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/embeddings/token_type_embeddings/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/embeddings/token_type_embeddings/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/token_type_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 2
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings/token_type_embeddings/adam_v"
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/token_type_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings/token_type_embeddings/adam_v"
  input: "electra/embeddings/token_type_embeddings/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/token_type_embeddings/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/token_type_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_28/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_28/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_28/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_28/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings/token_type_embeddings/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_28"
  op: "Mul"
  input: "Mul_28/x"
  input: "Mul_28/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_29/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_29"
  op: "Mul"
  input: "Mul_29/x"
  input: "CrossReplicaSum_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_16"
  op: "AddV2"
  input: "Mul_28"
  input: "Mul_29"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_30/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_30/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_30/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_30/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings/token_type_embeddings/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_30"
  op: "Mul"
  input: "Mul_30/x"
  input: "Mul_30/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_1"
  op: "Square"
  input: "CrossReplicaSum_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_31/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_31"
  op: "Mul"
  input: "Mul_31/x"
  input: "Square_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_17"
  op: "AddV2"
  input: "Mul_30"
  input: "Mul_31"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_1"
  op: "Sqrt"
  input: "add_17"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_18/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_18"
  op: "AddV2"
  input: "Sqrt_1"
  input: "add_18/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_4"
  op: "RealDiv"
  input: "add_16"
  input: "add_18"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_5"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_32/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_32"
  op: "Mul"
  input: "mul_32/x"
  input: "ReadVariableOp_5"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_19"
  op: "AddV2"
  input: "truediv_4"
  input: "mul_32"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_33"
  op: "Mul"
  input: "mul_21"
  input: "add_19"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_6"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_5"
  op: "Sub"
  input: "ReadVariableOp_6"
  input: "mul_33"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_3"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/MatMul/ReadVariableOp/Enter"
  input: "sub_5"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_7"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_3"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_4"
  op: "AssignVariableOp"
  input: "Mul_28/ReadVariableOp/Enter"
  input: "add_16"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_8"
  op: "ReadVariableOp"
  input: "Mul_28/ReadVariableOp/Enter"
  input: "^AssignVariableOp_4"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_5"
  op: "AssignVariableOp"
  input: "Mul_30/ReadVariableOp/Enter"
  input: "add_17"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_9"
  op: "ReadVariableOp"
  input: "Mul_30/ReadVariableOp/Enter"
  input: "^AssignVariableOp_5"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/embeddings/position_embeddings/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/embeddings/position_embeddings/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 512
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings/position_embeddings/adam_m"
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/position_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings/position_embeddings/adam_m"
  input: "electra/embeddings/position_embeddings/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/position_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/embeddings/position_embeddings/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/embeddings/position_embeddings/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings/position_embeddings/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 512
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings/position_embeddings/adam_v"
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/position_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings/position_embeddings/adam_v"
  input: "electra/embeddings/position_embeddings/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings/position_embeddings/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/position_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_34/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_34/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_34/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_34/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings/position_embeddings/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_34"
  op: "Mul"
  input: "Mul_34/x"
  input: "Mul_34/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_35/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_35"
  op: "Mul"
  input: "Mul_35/x"
  input: "CrossReplicaSum_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_20"
  op: "AddV2"
  input: "Mul_34"
  input: "Mul_35"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_36/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_36/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_36/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_36/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings/position_embeddings/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_36"
  op: "Mul"
  input: "Mul_36/x"
  input: "Mul_36/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_2"
  op: "Square"
  input: "CrossReplicaSum_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_37/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_37"
  op: "Mul"
  input: "Mul_37/x"
  input: "Square_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_21"
  op: "AddV2"
  input: "Mul_36"
  input: "Mul_37"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_2"
  op: "Sqrt"
  input: "add_21"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_22/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_22"
  op: "AddV2"
  input: "Sqrt_2"
  input: "add_22/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_5"
  op: "RealDiv"
  input: "add_20"
  input: "add_22"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_10"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/Slice/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_38/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_38"
  op: "Mul"
  input: "mul_38/x"
  input: "ReadVariableOp_10"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_23"
  op: "AddV2"
  input: "truediv_5"
  input: "mul_38"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_39"
  op: "Mul"
  input: "mul_21"
  input: "add_23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_11"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/Slice/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_6"
  op: "Sub"
  input: "ReadVariableOp_11"
  input: "mul_39"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_6"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/Slice/ReadVariableOp/Enter"
  input: "sub_6"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_12"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/Slice/ReadVariableOp/Enter"
  input: "^AssignVariableOp_6"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_7"
  op: "AssignVariableOp"
  input: "Mul_34/ReadVariableOp/Enter"
  input: "add_20"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_13"
  op: "ReadVariableOp"
  input: "Mul_34/ReadVariableOp/Enter"
  input: "^AssignVariableOp_7"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_8"
  op: "AssignVariableOp"
  input: "Mul_36/ReadVariableOp/Enter"
  input: "add_21"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_14"
  op: "ReadVariableOp"
  input: "Mul_36/ReadVariableOp/Enter"
  input: "^AssignVariableOp_8"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_1/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_m"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_1/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_v"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_40/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_40/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_40/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_40/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_40"
  op: "Mul"
  input: "Mul_40/x"
  input: "Mul_40/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_41/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_41"
  op: "Mul"
  input: "Mul_41/x"
  input: "CrossReplicaSum_3"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_24"
  op: "AddV2"
  input: "Mul_40"
  input: "Mul_41"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_42/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_42/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_42/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_42/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_42"
  op: "Mul"
  input: "Mul_42/x"
  input: "Mul_42/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_3"
  op: "Square"
  input: "CrossReplicaSum_3"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_43/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_43"
  op: "Mul"
  input: "Mul_43/x"
  input: "Square_3"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_25"
  op: "AddV2"
  input: "Mul_42"
  input: "Mul_43"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_3"
  op: "Sqrt"
  input: "add_25"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_26/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_26"
  op: "AddV2"
  input: "Sqrt_3"
  input: "add_26/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_6"
  op: "RealDiv"
  input: "add_24"
  input: "add_26"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_44"
  op: "Mul"
  input: "mul_21"
  input: "truediv_6"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_15"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_7"
  op: "Sub"
  input: "ReadVariableOp_15"
  input: "mul_44"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_9"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_7"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_16"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_9"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_10"
  op: "AssignVariableOp"
  input: "Mul_40/ReadVariableOp/Enter"
  input: "add_24"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_17"
  op: "ReadVariableOp"
  input: "Mul_40/ReadVariableOp/Enter"
  input: "^AssignVariableOp_10"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_11"
  op: "AssignVariableOp"
  input: "Mul_42/ReadVariableOp/Enter"
  input: "add_25"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_18"
  op: "ReadVariableOp"
  input: "Mul_42/ReadVariableOp/Enter"
  input: "^AssignVariableOp_11"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_1/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_m"
  input: "electra/embeddings_1/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_1/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_1/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_v"
  input: "electra/embeddings_1/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_1/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_45/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_45/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_45/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_45/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_1/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_45"
  op: "Mul"
  input: "Mul_45/x"
  input: "Mul_45/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_46/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_46"
  op: "Mul"
  input: "Mul_46/x"
  input: "CrossReplicaSum_4"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_27"
  op: "AddV2"
  input: "Mul_45"
  input: "Mul_46"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_47/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_47/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_47/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_47/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_1/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_47"
  op: "Mul"
  input: "Mul_47/x"
  input: "Mul_47/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_4"
  op: "Square"
  input: "CrossReplicaSum_4"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_48/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_48"
  op: "Mul"
  input: "Mul_48/x"
  input: "Square_4"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_28"
  op: "AddV2"
  input: "Mul_47"
  input: "Mul_48"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_4"
  op: "Sqrt"
  input: "add_28"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_29/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_29"
  op: "AddV2"
  input: "Sqrt_4"
  input: "add_29/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_7"
  op: "RealDiv"
  input: "add_27"
  input: "add_29"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_49"
  op: "Mul"
  input: "mul_21"
  input: "truediv_7"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_19"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_8"
  op: "Sub"
  input: "ReadVariableOp_19"
  input: "mul_49"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_12"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_8"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_20"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_12"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_13"
  op: "AssignVariableOp"
  input: "Mul_45/ReadVariableOp/Enter"
  input: "add_27"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_21"
  op: "ReadVariableOp"
  input: "Mul_45/ReadVariableOp/Enter"
  input: "^AssignVariableOp_13"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_14"
  op: "AssignVariableOp"
  input: "Mul_47/ReadVariableOp/Enter"
  input: "add_28"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_22"
  op: "ReadVariableOp"
  input: "Mul_47/ReadVariableOp/Enter"
  input: "^AssignVariableOp_14"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/embeddings_project/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/embeddings_project/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/embeddings_project/kernel/adam_m"
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/kernel/adam_m"
  input: "generator/embeddings_project/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/embeddings_project/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/embeddings_project/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/embeddings_project/kernel/adam_v"
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/kernel/adam_v"
  input: "generator/embeddings_project/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_50/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_50/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_50/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_50/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/embeddings_project/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_50"
  op: "Mul"
  input: "Mul_50/x"
  input: "Mul_50/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_51/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_51"
  op: "Mul"
  input: "Mul_51/x"
  input: "CrossReplicaSum_5"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_30"
  op: "AddV2"
  input: "Mul_50"
  input: "Mul_51"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_52/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_52/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_52/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_52/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/embeddings_project/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_52"
  op: "Mul"
  input: "Mul_52/x"
  input: "Mul_52/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_5"
  op: "Square"
  input: "CrossReplicaSum_5"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_53/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_53"
  op: "Mul"
  input: "Mul_53/x"
  input: "Square_5"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_31"
  op: "AddV2"
  input: "Mul_52"
  input: "Mul_53"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_5"
  op: "Sqrt"
  input: "add_31"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_32/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_32"
  op: "AddV2"
  input: "Sqrt_5"
  input: "add_32/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_8"
  op: "RealDiv"
  input: "add_30"
  input: "add_32"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_23"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_54/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_54"
  op: "Mul"
  input: "mul_54/x"
  input: "ReadVariableOp_23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_33"
  op: "AddV2"
  input: "truediv_8"
  input: "mul_54"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_55"
  op: "Mul"
  input: "mul_21"
  input: "add_33"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_24"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_9"
  op: "Sub"
  input: "ReadVariableOp_24"
  input: "mul_55"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_15"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/Tensordot/ReadVariableOp/Enter"
  input: "sub_9"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_25"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/Tensordot/ReadVariableOp/Enter"
  input: "^AssignVariableOp_15"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_16"
  op: "AssignVariableOp"
  input: "Mul_50/ReadVariableOp/Enter"
  input: "add_30"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_26"
  op: "ReadVariableOp"
  input: "Mul_50/ReadVariableOp/Enter"
  input: "^AssignVariableOp_16"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_17"
  op: "AssignVariableOp"
  input: "Mul_52/ReadVariableOp/Enter"
  input: "add_31"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_27"
  op: "ReadVariableOp"
  input: "Mul_52/ReadVariableOp/Enter"
  input: "^AssignVariableOp_17"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/embeddings_project/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/embeddings_project/bias/adam_m"
    }
  }
}
node {
  name: "generator/embeddings_project/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/embeddings_project/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/bias/adam_m"
  input: "generator/embeddings_project/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/embeddings_project/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/embeddings_project/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/embeddings_project/bias/adam_v"
    }
  }
}
node {
  name: "generator/embeddings_project/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/embeddings_project/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/bias/adam_v"
  input: "generator/embeddings_project/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/embeddings_project/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_56/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_56/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_56/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_56/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/embeddings_project/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_56"
  op: "Mul"
  input: "Mul_56/x"
  input: "Mul_56/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_57/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_57"
  op: "Mul"
  input: "Mul_57/x"
  input: "CrossReplicaSum_6"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_34"
  op: "AddV2"
  input: "Mul_56"
  input: "Mul_57"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_58/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_58/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_58/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_58/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/embeddings_project/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_58"
  op: "Mul"
  input: "Mul_58/x"
  input: "Mul_58/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_6"
  op: "Square"
  input: "CrossReplicaSum_6"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_59/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_59"
  op: "Mul"
  input: "Mul_59/x"
  input: "Square_6"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_35"
  op: "AddV2"
  input: "Mul_58"
  input: "Mul_59"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_6"
  op: "Sqrt"
  input: "add_35"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_36/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_36"
  op: "AddV2"
  input: "Sqrt_6"
  input: "add_36/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_9"
  op: "RealDiv"
  input: "add_34"
  input: "add_36"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_60"
  op: "Mul"
  input: "mul_21"
  input: "truediv_9"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_28"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_10"
  op: "Sub"
  input: "ReadVariableOp_28"
  input: "mul_60"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_18"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/BiasAdd/ReadVariableOp/Enter"
  input: "sub_10"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_29"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_18"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_19"
  op: "AssignVariableOp"
  input: "Mul_56/ReadVariableOp/Enter"
  input: "add_34"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_30"
  op: "ReadVariableOp"
  input: "Mul_56/ReadVariableOp/Enter"
  input: "^AssignVariableOp_19"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_20"
  op: "AssignVariableOp"
  input: "Mul_58/ReadVariableOp/Enter"
  input: "add_35"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_31"
  op: "ReadVariableOp"
  input: "Mul_58/ReadVariableOp/Enter"
  input: "^AssignVariableOp_20"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_61/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_61/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_61/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_61/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_61"
  op: "Mul"
  input: "Mul_61/x"
  input: "Mul_61/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_62/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_62"
  op: "Mul"
  input: "Mul_62/x"
  input: "CrossReplicaSum_7"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_37"
  op: "AddV2"
  input: "Mul_61"
  input: "Mul_62"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_63/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_63/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_63/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_63/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_63"
  op: "Mul"
  input: "Mul_63/x"
  input: "Mul_63/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_7"
  op: "Square"
  input: "CrossReplicaSum_7"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_64/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_64"
  op: "Mul"
  input: "Mul_64/x"
  input: "Square_7"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_38"
  op: "AddV2"
  input: "Mul_63"
  input: "Mul_64"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_7"
  op: "Sqrt"
  input: "add_38"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_39/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_39"
  op: "AddV2"
  input: "Sqrt_7"
  input: "add_39/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_10"
  op: "RealDiv"
  input: "add_37"
  input: "add_39"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_32"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_65/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_65"
  op: "Mul"
  input: "mul_65/x"
  input: "ReadVariableOp_32"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_40"
  op: "AddV2"
  input: "truediv_10"
  input: "mul_65"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_66"
  op: "Mul"
  input: "mul_21"
  input: "add_40"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_33"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_11"
  op: "Sub"
  input: "ReadVariableOp_33"
  input: "mul_66"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_21"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_11"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_34"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_21"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_22"
  op: "AssignVariableOp"
  input: "Mul_61/ReadVariableOp/Enter"
  input: "add_37"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_35"
  op: "ReadVariableOp"
  input: "Mul_61/ReadVariableOp/Enter"
  input: "^AssignVariableOp_22"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_23"
  op: "AssignVariableOp"
  input: "Mul_63/ReadVariableOp/Enter"
  input: "add_38"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_36"
  op: "ReadVariableOp"
  input: "Mul_63/ReadVariableOp/Enter"
  input: "^AssignVariableOp_23"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_67/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_67/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_67/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_67/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_67"
  op: "Mul"
  input: "Mul_67/x"
  input: "Mul_67/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_68/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_68"
  op: "Mul"
  input: "Mul_68/x"
  input: "CrossReplicaSum_8"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_41"
  op: "AddV2"
  input: "Mul_67"
  input: "Mul_68"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_69/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_69/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_69/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_69/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_69"
  op: "Mul"
  input: "Mul_69/x"
  input: "Mul_69/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_8"
  op: "Square"
  input: "CrossReplicaSum_8"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_70/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_70"
  op: "Mul"
  input: "Mul_70/x"
  input: "Square_8"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_42"
  op: "AddV2"
  input: "Mul_69"
  input: "Mul_70"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_8"
  op: "Sqrt"
  input: "add_42"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_43/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_43"
  op: "AddV2"
  input: "Sqrt_8"
  input: "add_43/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_11"
  op: "RealDiv"
  input: "add_41"
  input: "add_43"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_71"
  op: "Mul"
  input: "mul_21"
  input: "truediv_11"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_37"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_12"
  op: "Sub"
  input: "ReadVariableOp_37"
  input: "mul_71"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_24"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_12"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_38"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_24"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_25"
  op: "AssignVariableOp"
  input: "Mul_67/ReadVariableOp/Enter"
  input: "add_41"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_39"
  op: "ReadVariableOp"
  input: "Mul_67/ReadVariableOp/Enter"
  input: "^AssignVariableOp_25"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_26"
  op: "AssignVariableOp"
  input: "Mul_69/ReadVariableOp/Enter"
  input: "add_42"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_40"
  op: "ReadVariableOp"
  input: "Mul_69/ReadVariableOp/Enter"
  input: "^AssignVariableOp_26"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_72/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_72/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_72/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_72/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_72"
  op: "Mul"
  input: "Mul_72/x"
  input: "Mul_72/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_73/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_73"
  op: "Mul"
  input: "Mul_73/x"
  input: "CrossReplicaSum_9"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_44"
  op: "AddV2"
  input: "Mul_72"
  input: "Mul_73"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_74/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_74/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_74/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_74/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_74"
  op: "Mul"
  input: "Mul_74/x"
  input: "Mul_74/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_9"
  op: "Square"
  input: "CrossReplicaSum_9"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_75/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_75"
  op: "Mul"
  input: "Mul_75/x"
  input: "Square_9"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_45"
  op: "AddV2"
  input: "Mul_74"
  input: "Mul_75"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_9"
  op: "Sqrt"
  input: "add_45"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_46/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_46"
  op: "AddV2"
  input: "Sqrt_9"
  input: "add_46/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_12"
  op: "RealDiv"
  input: "add_44"
  input: "add_46"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_41"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_76/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_76"
  op: "Mul"
  input: "mul_76/x"
  input: "ReadVariableOp_41"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_47"
  op: "AddV2"
  input: "truediv_12"
  input: "mul_76"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_77"
  op: "Mul"
  input: "mul_21"
  input: "add_47"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_42"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_13"
  op: "Sub"
  input: "ReadVariableOp_42"
  input: "mul_77"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_27"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_13"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_43"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_27"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_28"
  op: "AssignVariableOp"
  input: "Mul_72/ReadVariableOp/Enter"
  input: "add_44"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_44"
  op: "ReadVariableOp"
  input: "Mul_72/ReadVariableOp/Enter"
  input: "^AssignVariableOp_28"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_29"
  op: "AssignVariableOp"
  input: "Mul_74/ReadVariableOp/Enter"
  input: "add_45"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_45"
  op: "ReadVariableOp"
  input: "Mul_74/ReadVariableOp/Enter"
  input: "^AssignVariableOp_29"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_78/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_78/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_78/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_78/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_78"
  op: "Mul"
  input: "Mul_78/x"
  input: "Mul_78/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_79/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_79"
  op: "Mul"
  input: "Mul_79/x"
  input: "CrossReplicaSum_10"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_48"
  op: "AddV2"
  input: "Mul_78"
  input: "Mul_79"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_80/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_80/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_80/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_80/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_80"
  op: "Mul"
  input: "Mul_80/x"
  input: "Mul_80/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_10"
  op: "Square"
  input: "CrossReplicaSum_10"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_81/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_81"
  op: "Mul"
  input: "Mul_81/x"
  input: "Square_10"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_49"
  op: "AddV2"
  input: "Mul_80"
  input: "Mul_81"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_10"
  op: "Sqrt"
  input: "add_49"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_50/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_50"
  op: "AddV2"
  input: "Sqrt_10"
  input: "add_50/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_13"
  op: "RealDiv"
  input: "add_48"
  input: "add_50"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_82"
  op: "Mul"
  input: "mul_21"
  input: "truediv_13"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_46"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_14"
  op: "Sub"
  input: "ReadVariableOp_46"
  input: "mul_82"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_30"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_14"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_47"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_30"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_31"
  op: "AssignVariableOp"
  input: "Mul_78/ReadVariableOp/Enter"
  input: "add_48"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_48"
  op: "ReadVariableOp"
  input: "Mul_78/ReadVariableOp/Enter"
  input: "^AssignVariableOp_31"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_32"
  op: "AssignVariableOp"
  input: "Mul_80/ReadVariableOp/Enter"
  input: "add_49"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_49"
  op: "ReadVariableOp"
  input: "Mul_80/ReadVariableOp/Enter"
  input: "^AssignVariableOp_32"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_83/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_83/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_83/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_83/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_83"
  op: "Mul"
  input: "Mul_83/x"
  input: "Mul_83/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_84/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_84"
  op: "Mul"
  input: "Mul_84/x"
  input: "CrossReplicaSum_11"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_51"
  op: "AddV2"
  input: "Mul_83"
  input: "Mul_84"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_85/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_85/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_85/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_85/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_85"
  op: "Mul"
  input: "Mul_85/x"
  input: "Mul_85/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_11"
  op: "Square"
  input: "CrossReplicaSum_11"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_86/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_86"
  op: "Mul"
  input: "Mul_86/x"
  input: "Square_11"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_52"
  op: "AddV2"
  input: "Mul_85"
  input: "Mul_86"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_11"
  op: "Sqrt"
  input: "add_52"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_53/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_53"
  op: "AddV2"
  input: "Sqrt_11"
  input: "add_53/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_14"
  op: "RealDiv"
  input: "add_51"
  input: "add_53"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_50"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_87/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_87"
  op: "Mul"
  input: "mul_87/x"
  input: "ReadVariableOp_50"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_54"
  op: "AddV2"
  input: "truediv_14"
  input: "mul_87"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_88"
  op: "Mul"
  input: "mul_21"
  input: "add_54"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_51"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_15"
  op: "Sub"
  input: "ReadVariableOp_51"
  input: "mul_88"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_33"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_15"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_52"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_33"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_34"
  op: "AssignVariableOp"
  input: "Mul_83/ReadVariableOp/Enter"
  input: "add_51"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_53"
  op: "ReadVariableOp"
  input: "Mul_83/ReadVariableOp/Enter"
  input: "^AssignVariableOp_34"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_35"
  op: "AssignVariableOp"
  input: "Mul_85/ReadVariableOp/Enter"
  input: "add_52"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_54"
  op: "ReadVariableOp"
  input: "Mul_85/ReadVariableOp/Enter"
  input: "^AssignVariableOp_35"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_89/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_89/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_89/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_89/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_89"
  op: "Mul"
  input: "Mul_89/x"
  input: "Mul_89/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_90/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_90"
  op: "Mul"
  input: "Mul_90/x"
  input: "CrossReplicaSum_12"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_55"
  op: "AddV2"
  input: "Mul_89"
  input: "Mul_90"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_91/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_91/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_91/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_91/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_91"
  op: "Mul"
  input: "Mul_91/x"
  input: "Mul_91/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_12"
  op: "Square"
  input: "CrossReplicaSum_12"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_92/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_92"
  op: "Mul"
  input: "Mul_92/x"
  input: "Square_12"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_56"
  op: "AddV2"
  input: "Mul_91"
  input: "Mul_92"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_12"
  op: "Sqrt"
  input: "add_56"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_57/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_57"
  op: "AddV2"
  input: "Sqrt_12"
  input: "add_57/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_15"
  op: "RealDiv"
  input: "add_55"
  input: "add_57"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_93"
  op: "Mul"
  input: "mul_21"
  input: "truediv_15"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_55"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_16"
  op: "Sub"
  input: "ReadVariableOp_55"
  input: "mul_93"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_36"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_16"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_56"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_36"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_37"
  op: "AssignVariableOp"
  input: "Mul_89/ReadVariableOp/Enter"
  input: "add_55"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_57"
  op: "ReadVariableOp"
  input: "Mul_89/ReadVariableOp/Enter"
  input: "^AssignVariableOp_37"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_38"
  op: "AssignVariableOp"
  input: "Mul_91/ReadVariableOp/Enter"
  input: "add_56"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_58"
  op: "ReadVariableOp"
  input: "Mul_91/ReadVariableOp/Enter"
  input: "^AssignVariableOp_38"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_94/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_94/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_94/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_94/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_94"
  op: "Mul"
  input: "Mul_94/x"
  input: "Mul_94/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_95/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_95"
  op: "Mul"
  input: "Mul_95/x"
  input: "CrossReplicaSum_13"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_58"
  op: "AddV2"
  input: "Mul_94"
  input: "Mul_95"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_96/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_96/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_96/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_96/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_96"
  op: "Mul"
  input: "Mul_96/x"
  input: "Mul_96/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_13"
  op: "Square"
  input: "CrossReplicaSum_13"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_97/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_97"
  op: "Mul"
  input: "Mul_97/x"
  input: "Square_13"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_59"
  op: "AddV2"
  input: "Mul_96"
  input: "Mul_97"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_13"
  op: "Sqrt"
  input: "add_59"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_60/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_60"
  op: "AddV2"
  input: "Sqrt_13"
  input: "add_60/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_16"
  op: "RealDiv"
  input: "add_58"
  input: "add_60"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_59"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_98/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_98"
  op: "Mul"
  input: "mul_98/x"
  input: "ReadVariableOp_59"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_61"
  op: "AddV2"
  input: "truediv_16"
  input: "mul_98"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_99"
  op: "Mul"
  input: "mul_21"
  input: "add_61"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_60"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_17"
  op: "Sub"
  input: "ReadVariableOp_60"
  input: "mul_99"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_39"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_17"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_61"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_39"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_40"
  op: "AssignVariableOp"
  input: "Mul_94/ReadVariableOp/Enter"
  input: "add_58"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_62"
  op: "ReadVariableOp"
  input: "Mul_94/ReadVariableOp/Enter"
  input: "^AssignVariableOp_40"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_41"
  op: "AssignVariableOp"
  input: "Mul_96/ReadVariableOp/Enter"
  input: "add_59"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_63"
  op: "ReadVariableOp"
  input: "Mul_96/ReadVariableOp/Enter"
  input: "^AssignVariableOp_41"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_100/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_100/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_100/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_100/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_100"
  op: "Mul"
  input: "Mul_100/x"
  input: "Mul_100/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_101/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_101"
  op: "Mul"
  input: "Mul_101/x"
  input: "CrossReplicaSum_14"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_62"
  op: "AddV2"
  input: "Mul_100"
  input: "Mul_101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_102/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_102/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_102/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_102/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_102"
  op: "Mul"
  input: "Mul_102/x"
  input: "Mul_102/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_14"
  op: "Square"
  input: "CrossReplicaSum_14"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_103/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_103"
  op: "Mul"
  input: "Mul_103/x"
  input: "Square_14"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_63"
  op: "AddV2"
  input: "Mul_102"
  input: "Mul_103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_14"
  op: "Sqrt"
  input: "add_63"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_64/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_64"
  op: "AddV2"
  input: "Sqrt_14"
  input: "add_64/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_17"
  op: "RealDiv"
  input: "add_62"
  input: "add_64"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_104"
  op: "Mul"
  input: "mul_21"
  input: "truediv_17"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_64"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_18"
  op: "Sub"
  input: "ReadVariableOp_64"
  input: "mul_104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_42"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_18"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_65"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_42"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_43"
  op: "AssignVariableOp"
  input: "Mul_100/ReadVariableOp/Enter"
  input: "add_62"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_66"
  op: "ReadVariableOp"
  input: "Mul_100/ReadVariableOp/Enter"
  input: "^AssignVariableOp_43"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_44"
  op: "AssignVariableOp"
  input: "Mul_102/ReadVariableOp/Enter"
  input: "add_63"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_67"
  op: "ReadVariableOp"
  input: "Mul_102/ReadVariableOp/Enter"
  input: "^AssignVariableOp_44"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_105/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_105/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_105/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_105/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_105"
  op: "Mul"
  input: "Mul_105/x"
  input: "Mul_105/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_106/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_106"
  op: "Mul"
  input: "Mul_106/x"
  input: "CrossReplicaSum_15"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_65"
  op: "AddV2"
  input: "Mul_105"
  input: "Mul_106"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_107/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_107/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_107/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_107/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_107"
  op: "Mul"
  input: "Mul_107/x"
  input: "Mul_107/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_15"
  op: "Square"
  input: "CrossReplicaSum_15"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_108/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_108"
  op: "Mul"
  input: "Mul_108/x"
  input: "Square_15"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_66"
  op: "AddV2"
  input: "Mul_107"
  input: "Mul_108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_15"
  op: "Sqrt"
  input: "add_66"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_67/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_67"
  op: "AddV2"
  input: "Sqrt_15"
  input: "add_67/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_18"
  op: "RealDiv"
  input: "add_65"
  input: "add_67"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_109"
  op: "Mul"
  input: "mul_21"
  input: "truediv_18"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_68"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_19"
  op: "Sub"
  input: "ReadVariableOp_68"
  input: "mul_109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_45"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_19"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_69"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_45"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_46"
  op: "AssignVariableOp"
  input: "Mul_105/ReadVariableOp/Enter"
  input: "add_65"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_70"
  op: "ReadVariableOp"
  input: "Mul_105/ReadVariableOp/Enter"
  input: "^AssignVariableOp_46"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_47"
  op: "AssignVariableOp"
  input: "Mul_107/ReadVariableOp/Enter"
  input: "add_66"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_71"
  op: "ReadVariableOp"
  input: "Mul_107/ReadVariableOp/Enter"
  input: "^AssignVariableOp_47"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_110/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_110/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_110/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_110/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_110"
  op: "Mul"
  input: "Mul_110/x"
  input: "Mul_110/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_111/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_111"
  op: "Mul"
  input: "Mul_111/x"
  input: "CrossReplicaSum_16"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_68"
  op: "AddV2"
  input: "Mul_110"
  input: "Mul_111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_112/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_112/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_112/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_112/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_112"
  op: "Mul"
  input: "Mul_112/x"
  input: "Mul_112/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_16"
  op: "Square"
  input: "CrossReplicaSum_16"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_113/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_113"
  op: "Mul"
  input: "Mul_113/x"
  input: "Square_16"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_69"
  op: "AddV2"
  input: "Mul_112"
  input: "Mul_113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_16"
  op: "Sqrt"
  input: "add_69"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_70/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_70"
  op: "AddV2"
  input: "Sqrt_16"
  input: "add_70/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_19"
  op: "RealDiv"
  input: "add_68"
  input: "add_70"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_114"
  op: "Mul"
  input: "mul_21"
  input: "truediv_19"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_72"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_20"
  op: "Sub"
  input: "ReadVariableOp_72"
  input: "mul_114"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_48"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_20"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_73"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_48"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_49"
  op: "AssignVariableOp"
  input: "Mul_110/ReadVariableOp/Enter"
  input: "add_68"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_74"
  op: "ReadVariableOp"
  input: "Mul_110/ReadVariableOp/Enter"
  input: "^AssignVariableOp_49"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_50"
  op: "AssignVariableOp"
  input: "Mul_112/ReadVariableOp/Enter"
  input: "add_69"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_75"
  op: "ReadVariableOp"
  input: "Mul_112/ReadVariableOp/Enter"
  input: "^AssignVariableOp_50"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_115/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_115/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_115/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_115/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_115"
  op: "Mul"
  input: "Mul_115/x"
  input: "Mul_115/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_116/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_116"
  op: "Mul"
  input: "Mul_116/x"
  input: "CrossReplicaSum_17"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_71"
  op: "AddV2"
  input: "Mul_115"
  input: "Mul_116"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_117/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_117/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_117/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_117/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_117"
  op: "Mul"
  input: "Mul_117/x"
  input: "Mul_117/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_17"
  op: "Square"
  input: "CrossReplicaSum_17"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_118/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_118"
  op: "Mul"
  input: "Mul_118/x"
  input: "Square_17"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_72"
  op: "AddV2"
  input: "Mul_117"
  input: "Mul_118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_17"
  op: "Sqrt"
  input: "add_72"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_73/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_73"
  op: "AddV2"
  input: "Sqrt_17"
  input: "add_73/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_20"
  op: "RealDiv"
  input: "add_71"
  input: "add_73"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_76"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_119/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_119"
  op: "Mul"
  input: "mul_119/x"
  input: "ReadVariableOp_76"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_74"
  op: "AddV2"
  input: "truediv_20"
  input: "mul_119"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_120"
  op: "Mul"
  input: "mul_21"
  input: "add_74"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_77"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_21"
  op: "Sub"
  input: "ReadVariableOp_77"
  input: "mul_120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_51"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_21"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_78"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_51"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_52"
  op: "AssignVariableOp"
  input: "Mul_115/ReadVariableOp/Enter"
  input: "add_71"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_79"
  op: "ReadVariableOp"
  input: "Mul_115/ReadVariableOp/Enter"
  input: "^AssignVariableOp_52"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_53"
  op: "AssignVariableOp"
  input: "Mul_117/ReadVariableOp/Enter"
  input: "add_72"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_80"
  op: "ReadVariableOp"
  input: "Mul_117/ReadVariableOp/Enter"
  input: "^AssignVariableOp_53"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_121/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_121/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_121/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_121/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_121"
  op: "Mul"
  input: "Mul_121/x"
  input: "Mul_121/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_122/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_122"
  op: "Mul"
  input: "Mul_122/x"
  input: "CrossReplicaSum_18"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_75"
  op: "AddV2"
  input: "Mul_121"
  input: "Mul_122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_123/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_123/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_123/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_123/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_123"
  op: "Mul"
  input: "Mul_123/x"
  input: "Mul_123/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_18"
  op: "Square"
  input: "CrossReplicaSum_18"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_124/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_124"
  op: "Mul"
  input: "Mul_124/x"
  input: "Square_18"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_76"
  op: "AddV2"
  input: "Mul_123"
  input: "Mul_124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_18"
  op: "Sqrt"
  input: "add_76"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_77/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_77"
  op: "AddV2"
  input: "Sqrt_18"
  input: "add_77/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_21"
  op: "RealDiv"
  input: "add_75"
  input: "add_77"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_125"
  op: "Mul"
  input: "mul_21"
  input: "truediv_21"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_81"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_22"
  op: "Sub"
  input: "ReadVariableOp_81"
  input: "mul_125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_54"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_22"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_82"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_54"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_55"
  op: "AssignVariableOp"
  input: "Mul_121/ReadVariableOp/Enter"
  input: "add_75"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_83"
  op: "ReadVariableOp"
  input: "Mul_121/ReadVariableOp/Enter"
  input: "^AssignVariableOp_55"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_56"
  op: "AssignVariableOp"
  input: "Mul_123/ReadVariableOp/Enter"
  input: "add_76"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_84"
  op: "ReadVariableOp"
  input: "Mul_123/ReadVariableOp/Enter"
  input: "^AssignVariableOp_56"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_126/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_126/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_126/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_126/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_126"
  op: "Mul"
  input: "Mul_126/x"
  input: "Mul_126/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_127/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_127"
  op: "Mul"
  input: "Mul_127/x"
  input: "CrossReplicaSum_19"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_78"
  op: "AddV2"
  input: "Mul_126"
  input: "Mul_127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_128/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_128/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_128/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_128/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_128"
  op: "Mul"
  input: "Mul_128/x"
  input: "Mul_128/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_19"
  op: "Square"
  input: "CrossReplicaSum_19"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_129/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_129"
  op: "Mul"
  input: "Mul_129/x"
  input: "Square_19"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_79"
  op: "AddV2"
  input: "Mul_128"
  input: "Mul_129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_19"
  op: "Sqrt"
  input: "add_79"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_80/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_80"
  op: "AddV2"
  input: "Sqrt_19"
  input: "add_80/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_22"
  op: "RealDiv"
  input: "add_78"
  input: "add_80"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_85"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_130/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_130"
  op: "Mul"
  input: "mul_130/x"
  input: "ReadVariableOp_85"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_81"
  op: "AddV2"
  input: "truediv_22"
  input: "mul_130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_131"
  op: "Mul"
  input: "mul_21"
  input: "add_81"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_86"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_23"
  op: "Sub"
  input: "ReadVariableOp_86"
  input: "mul_131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_57"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_23"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_87"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_57"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_58"
  op: "AssignVariableOp"
  input: "Mul_126/ReadVariableOp/Enter"
  input: "add_78"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_88"
  op: "ReadVariableOp"
  input: "Mul_126/ReadVariableOp/Enter"
  input: "^AssignVariableOp_58"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_59"
  op: "AssignVariableOp"
  input: "Mul_128/ReadVariableOp/Enter"
  input: "add_79"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_89"
  op: "ReadVariableOp"
  input: "Mul_128/ReadVariableOp/Enter"
  input: "^AssignVariableOp_59"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_m"
  input: "generator/encoder/layer_0/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_v"
  input: "generator/encoder/layer_0/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_132/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_132/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_132/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_132/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_132"
  op: "Mul"
  input: "Mul_132/x"
  input: "Mul_132/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_133/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_133"
  op: "Mul"
  input: "Mul_133/x"
  input: "CrossReplicaSum_20"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_82"
  op: "AddV2"
  input: "Mul_132"
  input: "Mul_133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_134/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_134/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_134/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_134/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_134"
  op: "Mul"
  input: "Mul_134/x"
  input: "Mul_134/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_20"
  op: "Square"
  input: "CrossReplicaSum_20"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_135/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_135"
  op: "Mul"
  input: "Mul_135/x"
  input: "Square_20"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_83"
  op: "AddV2"
  input: "Mul_134"
  input: "Mul_135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_20"
  op: "Sqrt"
  input: "add_83"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_84/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_84"
  op: "AddV2"
  input: "Sqrt_20"
  input: "add_84/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_23"
  op: "RealDiv"
  input: "add_82"
  input: "add_84"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_136"
  op: "Mul"
  input: "mul_21"
  input: "truediv_23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_90"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_24"
  op: "Sub"
  input: "ReadVariableOp_90"
  input: "mul_136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_60"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_24"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_91"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_60"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_61"
  op: "AssignVariableOp"
  input: "Mul_132/ReadVariableOp/Enter"
  input: "add_82"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_92"
  op: "ReadVariableOp"
  input: "Mul_132/ReadVariableOp/Enter"
  input: "^AssignVariableOp_61"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_62"
  op: "AssignVariableOp"
  input: "Mul_134/ReadVariableOp/Enter"
  input: "add_83"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_93"
  op: "ReadVariableOp"
  input: "Mul_134/ReadVariableOp/Enter"
  input: "^AssignVariableOp_62"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_137/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_137/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_137/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_137/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_137"
  op: "Mul"
  input: "Mul_137/x"
  input: "Mul_137/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_138/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_138"
  op: "Mul"
  input: "Mul_138/x"
  input: "CrossReplicaSum_21"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_85"
  op: "AddV2"
  input: "Mul_137"
  input: "Mul_138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_139/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_139/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_139/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_139/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_139"
  op: "Mul"
  input: "Mul_139/x"
  input: "Mul_139/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_21"
  op: "Square"
  input: "CrossReplicaSum_21"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_140/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_140"
  op: "Mul"
  input: "Mul_140/x"
  input: "Square_21"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_86"
  op: "AddV2"
  input: "Mul_139"
  input: "Mul_140"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_21"
  op: "Sqrt"
  input: "add_86"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_87/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_87"
  op: "AddV2"
  input: "Sqrt_21"
  input: "add_87/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_24"
  op: "RealDiv"
  input: "add_85"
  input: "add_87"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_141"
  op: "Mul"
  input: "mul_21"
  input: "truediv_24"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_94"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_25"
  op: "Sub"
  input: "ReadVariableOp_94"
  input: "mul_141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_63"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_25"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_95"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_63"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_64"
  op: "AssignVariableOp"
  input: "Mul_137/ReadVariableOp/Enter"
  input: "add_85"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_96"
  op: "ReadVariableOp"
  input: "Mul_137/ReadVariableOp/Enter"
  input: "^AssignVariableOp_64"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_65"
  op: "AssignVariableOp"
  input: "Mul_139/ReadVariableOp/Enter"
  input: "add_86"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_97"
  op: "ReadVariableOp"
  input: "Mul_139/ReadVariableOp/Enter"
  input: "^AssignVariableOp_65"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_142/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_142/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_142/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_142/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_142"
  op: "Mul"
  input: "Mul_142/x"
  input: "Mul_142/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_143/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_143"
  op: "Mul"
  input: "Mul_143/x"
  input: "CrossReplicaSum_22"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_88"
  op: "AddV2"
  input: "Mul_142"
  input: "Mul_143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_144/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_144/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_144/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_144/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_144"
  op: "Mul"
  input: "Mul_144/x"
  input: "Mul_144/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_22"
  op: "Square"
  input: "CrossReplicaSum_22"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_145/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_145"
  op: "Mul"
  input: "Mul_145/x"
  input: "Square_22"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_89"
  op: "AddV2"
  input: "Mul_144"
  input: "Mul_145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_22"
  op: "Sqrt"
  input: "add_89"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_90/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_90"
  op: "AddV2"
  input: "Sqrt_22"
  input: "add_90/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_25"
  op: "RealDiv"
  input: "add_88"
  input: "add_90"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_146"
  op: "Mul"
  input: "mul_21"
  input: "truediv_25"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_98"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_26"
  op: "Sub"
  input: "ReadVariableOp_98"
  input: "mul_146"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_66"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_26"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_99"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_66"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_67"
  op: "AssignVariableOp"
  input: "Mul_142/ReadVariableOp/Enter"
  input: "add_88"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_100"
  op: "ReadVariableOp"
  input: "Mul_142/ReadVariableOp/Enter"
  input: "^AssignVariableOp_67"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_68"
  op: "AssignVariableOp"
  input: "Mul_144/ReadVariableOp/Enter"
  input: "add_89"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_101"
  op: "ReadVariableOp"
  input: "Mul_144/ReadVariableOp/Enter"
  input: "^AssignVariableOp_68"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_147/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_147/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_147/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_147/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_147"
  op: "Mul"
  input: "Mul_147/x"
  input: "Mul_147/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_148/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_148"
  op: "Mul"
  input: "Mul_148/x"
  input: "CrossReplicaSum_23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_91"
  op: "AddV2"
  input: "Mul_147"
  input: "Mul_148"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_149/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_149/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_149/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_149/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_149"
  op: "Mul"
  input: "Mul_149/x"
  input: "Mul_149/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_23"
  op: "Square"
  input: "CrossReplicaSum_23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_150/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_150"
  op: "Mul"
  input: "Mul_150/x"
  input: "Square_23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_92"
  op: "AddV2"
  input: "Mul_149"
  input: "Mul_150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_23"
  op: "Sqrt"
  input: "add_92"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_93/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_93"
  op: "AddV2"
  input: "Sqrt_23"
  input: "add_93/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_26"
  op: "RealDiv"
  input: "add_91"
  input: "add_93"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_102"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_151/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_151"
  op: "Mul"
  input: "mul_151/x"
  input: "ReadVariableOp_102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_94"
  op: "AddV2"
  input: "truediv_26"
  input: "mul_151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_152"
  op: "Mul"
  input: "mul_21"
  input: "add_94"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_103"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_27"
  op: "Sub"
  input: "ReadVariableOp_103"
  input: "mul_152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_69"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_27"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_104"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_69"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_70"
  op: "AssignVariableOp"
  input: "Mul_147/ReadVariableOp/Enter"
  input: "add_91"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_105"
  op: "ReadVariableOp"
  input: "Mul_147/ReadVariableOp/Enter"
  input: "^AssignVariableOp_70"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_71"
  op: "AssignVariableOp"
  input: "Mul_149/ReadVariableOp/Enter"
  input: "add_92"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_106"
  op: "ReadVariableOp"
  input: "Mul_149/ReadVariableOp/Enter"
  input: "^AssignVariableOp_71"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_153/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_153/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_153/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_153/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_153"
  op: "Mul"
  input: "Mul_153/x"
  input: "Mul_153/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_154/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_154"
  op: "Mul"
  input: "Mul_154/x"
  input: "CrossReplicaSum_24"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_95"
  op: "AddV2"
  input: "Mul_153"
  input: "Mul_154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_155/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_155/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_155/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_155/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_155"
  op: "Mul"
  input: "Mul_155/x"
  input: "Mul_155/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_24"
  op: "Square"
  input: "CrossReplicaSum_24"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_156/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_156"
  op: "Mul"
  input: "Mul_156/x"
  input: "Square_24"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_96"
  op: "AddV2"
  input: "Mul_155"
  input: "Mul_156"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_24"
  op: "Sqrt"
  input: "add_96"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_97/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_97"
  op: "AddV2"
  input: "Sqrt_24"
  input: "add_97/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_27"
  op: "RealDiv"
  input: "add_95"
  input: "add_97"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_157"
  op: "Mul"
  input: "mul_21"
  input: "truediv_27"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_107"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_28"
  op: "Sub"
  input: "ReadVariableOp_107"
  input: "mul_157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_72"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_28"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_108"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_72"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_73"
  op: "AssignVariableOp"
  input: "Mul_153/ReadVariableOp/Enter"
  input: "add_95"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_109"
  op: "ReadVariableOp"
  input: "Mul_153/ReadVariableOp/Enter"
  input: "^AssignVariableOp_73"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_74"
  op: "AssignVariableOp"
  input: "Mul_155/ReadVariableOp/Enter"
  input: "add_96"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_110"
  op: "ReadVariableOp"
  input: "Mul_155/ReadVariableOp/Enter"
  input: "^AssignVariableOp_74"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_158/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_158/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_158/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_158/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_158"
  op: "Mul"
  input: "Mul_158/x"
  input: "Mul_158/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_159/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_159"
  op: "Mul"
  input: "Mul_159/x"
  input: "CrossReplicaSum_25"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_98"
  op: "AddV2"
  input: "Mul_158"
  input: "Mul_159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_160/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_160/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_160/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_160/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_160"
  op: "Mul"
  input: "Mul_160/x"
  input: "Mul_160/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_25"
  op: "Square"
  input: "CrossReplicaSum_25"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_161/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_161"
  op: "Mul"
  input: "Mul_161/x"
  input: "Square_25"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_99"
  op: "AddV2"
  input: "Mul_160"
  input: "Mul_161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_25"
  op: "Sqrt"
  input: "add_99"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_100/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_100"
  op: "AddV2"
  input: "Sqrt_25"
  input: "add_100/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_28"
  op: "RealDiv"
  input: "add_98"
  input: "add_100"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_111"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_162/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_162"
  op: "Mul"
  input: "mul_162/x"
  input: "ReadVariableOp_111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_101"
  op: "AddV2"
  input: "truediv_28"
  input: "mul_162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_163"
  op: "Mul"
  input: "mul_21"
  input: "add_101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_112"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_29"
  op: "Sub"
  input: "ReadVariableOp_112"
  input: "mul_163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_75"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_29"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_113"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_75"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_76"
  op: "AssignVariableOp"
  input: "Mul_158/ReadVariableOp/Enter"
  input: "add_98"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_114"
  op: "ReadVariableOp"
  input: "Mul_158/ReadVariableOp/Enter"
  input: "^AssignVariableOp_76"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_77"
  op: "AssignVariableOp"
  input: "Mul_160/ReadVariableOp/Enter"
  input: "add_99"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_115"
  op: "ReadVariableOp"
  input: "Mul_160/ReadVariableOp/Enter"
  input: "^AssignVariableOp_77"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_164/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_164/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_164/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_164/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_164"
  op: "Mul"
  input: "Mul_164/x"
  input: "Mul_164/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_165/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_165"
  op: "Mul"
  input: "Mul_165/x"
  input: "CrossReplicaSum_26"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_102"
  op: "AddV2"
  input: "Mul_164"
  input: "Mul_165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_166/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_166/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_166/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_166/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_166"
  op: "Mul"
  input: "Mul_166/x"
  input: "Mul_166/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_26"
  op: "Square"
  input: "CrossReplicaSum_26"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_167/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_167"
  op: "Mul"
  input: "Mul_167/x"
  input: "Square_26"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_103"
  op: "AddV2"
  input: "Mul_166"
  input: "Mul_167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_26"
  op: "Sqrt"
  input: "add_103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_104/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_104"
  op: "AddV2"
  input: "Sqrt_26"
  input: "add_104/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_29"
  op: "RealDiv"
  input: "add_102"
  input: "add_104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_168"
  op: "Mul"
  input: "mul_21"
  input: "truediv_29"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_116"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_30"
  op: "Sub"
  input: "ReadVariableOp_116"
  input: "mul_168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_78"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_30"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_117"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_78"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_79"
  op: "AssignVariableOp"
  input: "Mul_164/ReadVariableOp/Enter"
  input: "add_102"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_118"
  op: "ReadVariableOp"
  input: "Mul_164/ReadVariableOp/Enter"
  input: "^AssignVariableOp_79"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_80"
  op: "AssignVariableOp"
  input: "Mul_166/ReadVariableOp/Enter"
  input: "add_103"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_119"
  op: "ReadVariableOp"
  input: "Mul_166/ReadVariableOp/Enter"
  input: "^AssignVariableOp_80"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_169/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_169/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_169/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_169/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_169"
  op: "Mul"
  input: "Mul_169/x"
  input: "Mul_169/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_170/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_170"
  op: "Mul"
  input: "Mul_170/x"
  input: "CrossReplicaSum_27"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_105"
  op: "AddV2"
  input: "Mul_169"
  input: "Mul_170"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_171/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_171/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_171/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_171/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_171"
  op: "Mul"
  input: "Mul_171/x"
  input: "Mul_171/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_27"
  op: "Square"
  input: "CrossReplicaSum_27"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_172/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_172"
  op: "Mul"
  input: "Mul_172/x"
  input: "Square_27"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_106"
  op: "AddV2"
  input: "Mul_171"
  input: "Mul_172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_27"
  op: "Sqrt"
  input: "add_106"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_107/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_107"
  op: "AddV2"
  input: "Sqrt_27"
  input: "add_107/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_30"
  op: "RealDiv"
  input: "add_105"
  input: "add_107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_120"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_173/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_173"
  op: "Mul"
  input: "mul_173/x"
  input: "ReadVariableOp_120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_108"
  op: "AddV2"
  input: "truediv_30"
  input: "mul_173"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_174"
  op: "Mul"
  input: "mul_21"
  input: "add_108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_121"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_31"
  op: "Sub"
  input: "ReadVariableOp_121"
  input: "mul_174"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_81"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_31"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_122"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_81"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_82"
  op: "AssignVariableOp"
  input: "Mul_169/ReadVariableOp/Enter"
  input: "add_105"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_123"
  op: "ReadVariableOp"
  input: "Mul_169/ReadVariableOp/Enter"
  input: "^AssignVariableOp_82"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_83"
  op: "AssignVariableOp"
  input: "Mul_171/ReadVariableOp/Enter"
  input: "add_106"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_124"
  op: "ReadVariableOp"
  input: "Mul_171/ReadVariableOp/Enter"
  input: "^AssignVariableOp_83"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_175/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_175/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_175/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_175/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_175"
  op: "Mul"
  input: "Mul_175/x"
  input: "Mul_175/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_176/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_176"
  op: "Mul"
  input: "Mul_176/x"
  input: "CrossReplicaSum_28"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_109"
  op: "AddV2"
  input: "Mul_175"
  input: "Mul_176"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_177/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_177/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_177/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_177/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_177"
  op: "Mul"
  input: "Mul_177/x"
  input: "Mul_177/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_28"
  op: "Square"
  input: "CrossReplicaSum_28"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_178/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_178"
  op: "Mul"
  input: "Mul_178/x"
  input: "Square_28"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_110"
  op: "AddV2"
  input: "Mul_177"
  input: "Mul_178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_28"
  op: "Sqrt"
  input: "add_110"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_111/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_111"
  op: "AddV2"
  input: "Sqrt_28"
  input: "add_111/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_31"
  op: "RealDiv"
  input: "add_109"
  input: "add_111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_179"
  op: "Mul"
  input: "mul_21"
  input: "truediv_31"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_125"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_32"
  op: "Sub"
  input: "ReadVariableOp_125"
  input: "mul_179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_84"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_32"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_126"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_84"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_85"
  op: "AssignVariableOp"
  input: "Mul_175/ReadVariableOp/Enter"
  input: "add_109"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_127"
  op: "ReadVariableOp"
  input: "Mul_175/ReadVariableOp/Enter"
  input: "^AssignVariableOp_85"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_86"
  op: "AssignVariableOp"
  input: "Mul_177/ReadVariableOp/Enter"
  input: "add_110"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_128"
  op: "ReadVariableOp"
  input: "Mul_177/ReadVariableOp/Enter"
  input: "^AssignVariableOp_86"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_180/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_180/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_180/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_180/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_180"
  op: "Mul"
  input: "Mul_180/x"
  input: "Mul_180/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_181/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_181"
  op: "Mul"
  input: "Mul_181/x"
  input: "CrossReplicaSum_29"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_112"
  op: "AddV2"
  input: "Mul_180"
  input: "Mul_181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_182/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_182/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_182/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_182/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_182"
  op: "Mul"
  input: "Mul_182/x"
  input: "Mul_182/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_29"
  op: "Square"
  input: "CrossReplicaSum_29"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_183/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_183"
  op: "Mul"
  input: "Mul_183/x"
  input: "Square_29"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_113"
  op: "AddV2"
  input: "Mul_182"
  input: "Mul_183"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_29"
  op: "Sqrt"
  input: "add_113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_114/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_114"
  op: "AddV2"
  input: "Sqrt_29"
  input: "add_114/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_32"
  op: "RealDiv"
  input: "add_112"
  input: "add_114"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_129"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_184/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_184"
  op: "Mul"
  input: "mul_184/x"
  input: "ReadVariableOp_129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_115"
  op: "AddV2"
  input: "truediv_32"
  input: "mul_184"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_185"
  op: "Mul"
  input: "mul_21"
  input: "add_115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_130"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_33"
  op: "Sub"
  input: "ReadVariableOp_130"
  input: "mul_185"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_87"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_33"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_131"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_87"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_88"
  op: "AssignVariableOp"
  input: "Mul_180/ReadVariableOp/Enter"
  input: "add_112"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_132"
  op: "ReadVariableOp"
  input: "Mul_180/ReadVariableOp/Enter"
  input: "^AssignVariableOp_88"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_89"
  op: "AssignVariableOp"
  input: "Mul_182/ReadVariableOp/Enter"
  input: "add_113"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_133"
  op: "ReadVariableOp"
  input: "Mul_182/ReadVariableOp/Enter"
  input: "^AssignVariableOp_89"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_186/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_186/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_186/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_186/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_186"
  op: "Mul"
  input: "Mul_186/x"
  input: "Mul_186/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_187/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_187"
  op: "Mul"
  input: "Mul_187/x"
  input: "CrossReplicaSum_30"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_116"
  op: "AddV2"
  input: "Mul_186"
  input: "Mul_187"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_188/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_188/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_188/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_188/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_188"
  op: "Mul"
  input: "Mul_188/x"
  input: "Mul_188/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_30"
  op: "Square"
  input: "CrossReplicaSum_30"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_189/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_189"
  op: "Mul"
  input: "Mul_189/x"
  input: "Square_30"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_117"
  op: "AddV2"
  input: "Mul_188"
  input: "Mul_189"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_30"
  op: "Sqrt"
  input: "add_117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_118/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_118"
  op: "AddV2"
  input: "Sqrt_30"
  input: "add_118/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_33"
  op: "RealDiv"
  input: "add_116"
  input: "add_118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_190"
  op: "Mul"
  input: "mul_21"
  input: "truediv_33"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_134"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_34"
  op: "Sub"
  input: "ReadVariableOp_134"
  input: "mul_190"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_90"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_34"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_135"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_90"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_91"
  op: "AssignVariableOp"
  input: "Mul_186/ReadVariableOp/Enter"
  input: "add_116"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_136"
  op: "ReadVariableOp"
  input: "Mul_186/ReadVariableOp/Enter"
  input: "^AssignVariableOp_91"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_92"
  op: "AssignVariableOp"
  input: "Mul_188/ReadVariableOp/Enter"
  input: "add_117"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_137"
  op: "ReadVariableOp"
  input: "Mul_188/ReadVariableOp/Enter"
  input: "^AssignVariableOp_92"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_191/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_191/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_191/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_191/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_191"
  op: "Mul"
  input: "Mul_191/x"
  input: "Mul_191/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_192/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_192"
  op: "Mul"
  input: "Mul_192/x"
  input: "CrossReplicaSum_31"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_119"
  op: "AddV2"
  input: "Mul_191"
  input: "Mul_192"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_193/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_193/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_193/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_193/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_193"
  op: "Mul"
  input: "Mul_193/x"
  input: "Mul_193/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_31"
  op: "Square"
  input: "CrossReplicaSum_31"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_194/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_194"
  op: "Mul"
  input: "Mul_194/x"
  input: "Square_31"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_120"
  op: "AddV2"
  input: "Mul_193"
  input: "Mul_194"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_31"
  op: "Sqrt"
  input: "add_120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_121/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_121"
  op: "AddV2"
  input: "Sqrt_31"
  input: "add_121/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_34"
  op: "RealDiv"
  input: "add_119"
  input: "add_121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_195"
  op: "Mul"
  input: "mul_21"
  input: "truediv_34"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_138"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_35"
  op: "Sub"
  input: "ReadVariableOp_138"
  input: "mul_195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_93"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_35"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_139"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_93"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_94"
  op: "AssignVariableOp"
  input: "Mul_191/ReadVariableOp/Enter"
  input: "add_119"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_140"
  op: "ReadVariableOp"
  input: "Mul_191/ReadVariableOp/Enter"
  input: "^AssignVariableOp_94"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_95"
  op: "AssignVariableOp"
  input: "Mul_193/ReadVariableOp/Enter"
  input: "add_120"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_141"
  op: "ReadVariableOp"
  input: "Mul_193/ReadVariableOp/Enter"
  input: "^AssignVariableOp_95"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_196/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_196/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_196/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_196/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_196"
  op: "Mul"
  input: "Mul_196/x"
  input: "Mul_196/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_197/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_197"
  op: "Mul"
  input: "Mul_197/x"
  input: "CrossReplicaSum_32"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_122"
  op: "AddV2"
  input: "Mul_196"
  input: "Mul_197"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_198/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_198/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_198/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_198/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_198"
  op: "Mul"
  input: "Mul_198/x"
  input: "Mul_198/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_32"
  op: "Square"
  input: "CrossReplicaSum_32"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_199/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_199"
  op: "Mul"
  input: "Mul_199/x"
  input: "Square_32"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_123"
  op: "AddV2"
  input: "Mul_198"
  input: "Mul_199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_32"
  op: "Sqrt"
  input: "add_123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_124/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_124"
  op: "AddV2"
  input: "Sqrt_32"
  input: "add_124/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_35"
  op: "RealDiv"
  input: "add_122"
  input: "add_124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_200"
  op: "Mul"
  input: "mul_21"
  input: "truediv_35"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_142"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_36"
  op: "Sub"
  input: "ReadVariableOp_142"
  input: "mul_200"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_96"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_36"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_143"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_96"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_97"
  op: "AssignVariableOp"
  input: "Mul_196/ReadVariableOp/Enter"
  input: "add_122"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_144"
  op: "ReadVariableOp"
  input: "Mul_196/ReadVariableOp/Enter"
  input: "^AssignVariableOp_97"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_98"
  op: "AssignVariableOp"
  input: "Mul_198/ReadVariableOp/Enter"
  input: "add_123"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_145"
  op: "ReadVariableOp"
  input: "Mul_198/ReadVariableOp/Enter"
  input: "^AssignVariableOp_98"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_201/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_201/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_201/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_201/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_201"
  op: "Mul"
  input: "Mul_201/x"
  input: "Mul_201/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_202/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_202"
  op: "Mul"
  input: "Mul_202/x"
  input: "CrossReplicaSum_33"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_125"
  op: "AddV2"
  input: "Mul_201"
  input: "Mul_202"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_203/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_203/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_203/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_203/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_203"
  op: "Mul"
  input: "Mul_203/x"
  input: "Mul_203/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_33"
  op: "Square"
  input: "CrossReplicaSum_33"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_204/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_204"
  op: "Mul"
  input: "Mul_204/x"
  input: "Square_33"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_126"
  op: "AddV2"
  input: "Mul_203"
  input: "Mul_204"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_33"
  op: "Sqrt"
  input: "add_126"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_127/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_127"
  op: "AddV2"
  input: "Sqrt_33"
  input: "add_127/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_36"
  op: "RealDiv"
  input: "add_125"
  input: "add_127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_146"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_205/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_205"
  op: "Mul"
  input: "mul_205/x"
  input: "ReadVariableOp_146"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_128"
  op: "AddV2"
  input: "truediv_36"
  input: "mul_205"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_206"
  op: "Mul"
  input: "mul_21"
  input: "add_128"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_147"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_37"
  op: "Sub"
  input: "ReadVariableOp_147"
  input: "mul_206"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_99"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_37"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_148"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_99"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_100"
  op: "AssignVariableOp"
  input: "Mul_201/ReadVariableOp/Enter"
  input: "add_125"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_149"
  op: "ReadVariableOp"
  input: "Mul_201/ReadVariableOp/Enter"
  input: "^AssignVariableOp_100"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_101"
  op: "AssignVariableOp"
  input: "Mul_203/ReadVariableOp/Enter"
  input: "add_126"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_150"
  op: "ReadVariableOp"
  input: "Mul_203/ReadVariableOp/Enter"
  input: "^AssignVariableOp_101"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_207/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_207/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_207/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_207/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_207"
  op: "Mul"
  input: "Mul_207/x"
  input: "Mul_207/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_208/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_208"
  op: "Mul"
  input: "Mul_208/x"
  input: "CrossReplicaSum_34"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_129"
  op: "AddV2"
  input: "Mul_207"
  input: "Mul_208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_209/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_209/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_209/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_209/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_209"
  op: "Mul"
  input: "Mul_209/x"
  input: "Mul_209/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_34"
  op: "Square"
  input: "CrossReplicaSum_34"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_210/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_210"
  op: "Mul"
  input: "Mul_210/x"
  input: "Square_34"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_130"
  op: "AddV2"
  input: "Mul_209"
  input: "Mul_210"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_34"
  op: "Sqrt"
  input: "add_130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_131/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_131"
  op: "AddV2"
  input: "Sqrt_34"
  input: "add_131/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_37"
  op: "RealDiv"
  input: "add_129"
  input: "add_131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_211"
  op: "Mul"
  input: "mul_21"
  input: "truediv_37"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_151"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_38"
  op: "Sub"
  input: "ReadVariableOp_151"
  input: "mul_211"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_102"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_38"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_152"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_102"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_103"
  op: "AssignVariableOp"
  input: "Mul_207/ReadVariableOp/Enter"
  input: "add_129"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_153"
  op: "ReadVariableOp"
  input: "Mul_207/ReadVariableOp/Enter"
  input: "^AssignVariableOp_103"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_104"
  op: "AssignVariableOp"
  input: "Mul_209/ReadVariableOp/Enter"
  input: "add_130"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_154"
  op: "ReadVariableOp"
  input: "Mul_209/ReadVariableOp/Enter"
  input: "^AssignVariableOp_104"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_212/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_212/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_212/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_212/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_212"
  op: "Mul"
  input: "Mul_212/x"
  input: "Mul_212/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_213/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_213"
  op: "Mul"
  input: "Mul_213/x"
  input: "CrossReplicaSum_35"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_132"
  op: "AddV2"
  input: "Mul_212"
  input: "Mul_213"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_214/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_214/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_214/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_214/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_214"
  op: "Mul"
  input: "Mul_214/x"
  input: "Mul_214/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_35"
  op: "Square"
  input: "CrossReplicaSum_35"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_215/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_215"
  op: "Mul"
  input: "Mul_215/x"
  input: "Square_35"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_133"
  op: "AddV2"
  input: "Mul_214"
  input: "Mul_215"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_35"
  op: "Sqrt"
  input: "add_133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_134/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_134"
  op: "AddV2"
  input: "Sqrt_35"
  input: "add_134/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_38"
  op: "RealDiv"
  input: "add_132"
  input: "add_134"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_155"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_216/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_216"
  op: "Mul"
  input: "mul_216/x"
  input: "ReadVariableOp_155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_135"
  op: "AddV2"
  input: "truediv_38"
  input: "mul_216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_217"
  op: "Mul"
  input: "mul_21"
  input: "add_135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_156"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_39"
  op: "Sub"
  input: "ReadVariableOp_156"
  input: "mul_217"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_105"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_39"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_157"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_105"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_106"
  op: "AssignVariableOp"
  input: "Mul_212/ReadVariableOp/Enter"
  input: "add_132"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_158"
  op: "ReadVariableOp"
  input: "Mul_212/ReadVariableOp/Enter"
  input: "^AssignVariableOp_106"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_107"
  op: "AssignVariableOp"
  input: "Mul_214/ReadVariableOp/Enter"
  input: "add_133"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_159"
  op: "ReadVariableOp"
  input: "Mul_214/ReadVariableOp/Enter"
  input: "^AssignVariableOp_107"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_m"
  input: "generator/encoder/layer_1/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_v"
  input: "generator/encoder/layer_1/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_218/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_218/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_218/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_218/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_218"
  op: "Mul"
  input: "Mul_218/x"
  input: "Mul_218/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_219/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_219"
  op: "Mul"
  input: "Mul_219/x"
  input: "CrossReplicaSum_36"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_136"
  op: "AddV2"
  input: "Mul_218"
  input: "Mul_219"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_220/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_220/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_220/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_220/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_220"
  op: "Mul"
  input: "Mul_220/x"
  input: "Mul_220/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_36"
  op: "Square"
  input: "CrossReplicaSum_36"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_221/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_221"
  op: "Mul"
  input: "Mul_221/x"
  input: "Square_36"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_137"
  op: "AddV2"
  input: "Mul_220"
  input: "Mul_221"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_36"
  op: "Sqrt"
  input: "add_137"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_138/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_138"
  op: "AddV2"
  input: "Sqrt_36"
  input: "add_138/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_39"
  op: "RealDiv"
  input: "add_136"
  input: "add_138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_222"
  op: "Mul"
  input: "mul_21"
  input: "truediv_39"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_160"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_40"
  op: "Sub"
  input: "ReadVariableOp_160"
  input: "mul_222"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_108"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_40"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_161"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_108"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_109"
  op: "AssignVariableOp"
  input: "Mul_218/ReadVariableOp/Enter"
  input: "add_136"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_162"
  op: "ReadVariableOp"
  input: "Mul_218/ReadVariableOp/Enter"
  input: "^AssignVariableOp_109"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_110"
  op: "AssignVariableOp"
  input: "Mul_220/ReadVariableOp/Enter"
  input: "add_137"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_163"
  op: "ReadVariableOp"
  input: "Mul_220/ReadVariableOp/Enter"
  input: "^AssignVariableOp_110"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_223/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_223/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_223/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_223/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_223"
  op: "Mul"
  input: "Mul_223/x"
  input: "Mul_223/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_224/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_224"
  op: "Mul"
  input: "Mul_224/x"
  input: "CrossReplicaSum_37"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_139"
  op: "AddV2"
  input: "Mul_223"
  input: "Mul_224"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_225/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_225/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_225/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_225/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_225"
  op: "Mul"
  input: "Mul_225/x"
  input: "Mul_225/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_37"
  op: "Square"
  input: "CrossReplicaSum_37"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_226/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_226"
  op: "Mul"
  input: "Mul_226/x"
  input: "Square_37"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_140"
  op: "AddV2"
  input: "Mul_225"
  input: "Mul_226"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_37"
  op: "Sqrt"
  input: "add_140"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_141/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_141"
  op: "AddV2"
  input: "Sqrt_37"
  input: "add_141/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_40"
  op: "RealDiv"
  input: "add_139"
  input: "add_141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_227"
  op: "Mul"
  input: "mul_21"
  input: "truediv_40"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_164"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_41"
  op: "Sub"
  input: "ReadVariableOp_164"
  input: "mul_227"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_111"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_41"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_165"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_111"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_112"
  op: "AssignVariableOp"
  input: "Mul_223/ReadVariableOp/Enter"
  input: "add_139"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_166"
  op: "ReadVariableOp"
  input: "Mul_223/ReadVariableOp/Enter"
  input: "^AssignVariableOp_112"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_113"
  op: "AssignVariableOp"
  input: "Mul_225/ReadVariableOp/Enter"
  input: "add_140"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_167"
  op: "ReadVariableOp"
  input: "Mul_225/ReadVariableOp/Enter"
  input: "^AssignVariableOp_113"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_228/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_228/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_228/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_228/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_228"
  op: "Mul"
  input: "Mul_228/x"
  input: "Mul_228/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_229/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_229"
  op: "Mul"
  input: "Mul_229/x"
  input: "CrossReplicaSum_38"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_142"
  op: "AddV2"
  input: "Mul_228"
  input: "Mul_229"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_230/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_230/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_230/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_230/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_230"
  op: "Mul"
  input: "Mul_230/x"
  input: "Mul_230/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_38"
  op: "Square"
  input: "CrossReplicaSum_38"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_231/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_231"
  op: "Mul"
  input: "Mul_231/x"
  input: "Square_38"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_143"
  op: "AddV2"
  input: "Mul_230"
  input: "Mul_231"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_38"
  op: "Sqrt"
  input: "add_143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_144/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_144"
  op: "AddV2"
  input: "Sqrt_38"
  input: "add_144/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_41"
  op: "RealDiv"
  input: "add_142"
  input: "add_144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_232"
  op: "Mul"
  input: "mul_21"
  input: "truediv_41"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_168"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_42"
  op: "Sub"
  input: "ReadVariableOp_168"
  input: "mul_232"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_114"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_42"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_169"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_114"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_115"
  op: "AssignVariableOp"
  input: "Mul_228/ReadVariableOp/Enter"
  input: "add_142"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_170"
  op: "ReadVariableOp"
  input: "Mul_228/ReadVariableOp/Enter"
  input: "^AssignVariableOp_115"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_116"
  op: "AssignVariableOp"
  input: "Mul_230/ReadVariableOp/Enter"
  input: "add_143"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_171"
  op: "ReadVariableOp"
  input: "Mul_230/ReadVariableOp/Enter"
  input: "^AssignVariableOp_116"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_233/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_233/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_233/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_233/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_233"
  op: "Mul"
  input: "Mul_233/x"
  input: "Mul_233/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_234/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_234"
  op: "Mul"
  input: "Mul_234/x"
  input: "CrossReplicaSum_39"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_145"
  op: "AddV2"
  input: "Mul_233"
  input: "Mul_234"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_235/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_235/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_235/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_235/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_235"
  op: "Mul"
  input: "Mul_235/x"
  input: "Mul_235/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_39"
  op: "Square"
  input: "CrossReplicaSum_39"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_236/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_236"
  op: "Mul"
  input: "Mul_236/x"
  input: "Square_39"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_146"
  op: "AddV2"
  input: "Mul_235"
  input: "Mul_236"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_39"
  op: "Sqrt"
  input: "add_146"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_147/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_147"
  op: "AddV2"
  input: "Sqrt_39"
  input: "add_147/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_42"
  op: "RealDiv"
  input: "add_145"
  input: "add_147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_172"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_237/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_237"
  op: "Mul"
  input: "mul_237/x"
  input: "ReadVariableOp_172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_148"
  op: "AddV2"
  input: "truediv_42"
  input: "mul_237"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_238"
  op: "Mul"
  input: "mul_21"
  input: "add_148"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_173"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_43"
  op: "Sub"
  input: "ReadVariableOp_173"
  input: "mul_238"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_117"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_43"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_174"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_117"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_118"
  op: "AssignVariableOp"
  input: "Mul_233/ReadVariableOp/Enter"
  input: "add_145"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_175"
  op: "ReadVariableOp"
  input: "Mul_233/ReadVariableOp/Enter"
  input: "^AssignVariableOp_118"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_119"
  op: "AssignVariableOp"
  input: "Mul_235/ReadVariableOp/Enter"
  input: "add_146"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_176"
  op: "ReadVariableOp"
  input: "Mul_235/ReadVariableOp/Enter"
  input: "^AssignVariableOp_119"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_239/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_239/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_239/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_239/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_239"
  op: "Mul"
  input: "Mul_239/x"
  input: "Mul_239/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_240/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_240"
  op: "Mul"
  input: "Mul_240/x"
  input: "CrossReplicaSum_40"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_149"
  op: "AddV2"
  input: "Mul_239"
  input: "Mul_240"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_241/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_241/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_241/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_241/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_241"
  op: "Mul"
  input: "Mul_241/x"
  input: "Mul_241/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_40"
  op: "Square"
  input: "CrossReplicaSum_40"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_242/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_242"
  op: "Mul"
  input: "Mul_242/x"
  input: "Square_40"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_150"
  op: "AddV2"
  input: "Mul_241"
  input: "Mul_242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_40"
  op: "Sqrt"
  input: "add_150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_151/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_151"
  op: "AddV2"
  input: "Sqrt_40"
  input: "add_151/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_43"
  op: "RealDiv"
  input: "add_149"
  input: "add_151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_243"
  op: "Mul"
  input: "mul_21"
  input: "truediv_43"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_177"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_44"
  op: "Sub"
  input: "ReadVariableOp_177"
  input: "mul_243"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_120"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_44"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_178"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_120"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_121"
  op: "AssignVariableOp"
  input: "Mul_239/ReadVariableOp/Enter"
  input: "add_149"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_179"
  op: "ReadVariableOp"
  input: "Mul_239/ReadVariableOp/Enter"
  input: "^AssignVariableOp_121"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_122"
  op: "AssignVariableOp"
  input: "Mul_241/ReadVariableOp/Enter"
  input: "add_150"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_180"
  op: "ReadVariableOp"
  input: "Mul_241/ReadVariableOp/Enter"
  input: "^AssignVariableOp_122"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_244/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_244/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_244/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_244/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_244"
  op: "Mul"
  input: "Mul_244/x"
  input: "Mul_244/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_245/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_245"
  op: "Mul"
  input: "Mul_245/x"
  input: "CrossReplicaSum_41"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_152"
  op: "AddV2"
  input: "Mul_244"
  input: "Mul_245"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_246/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_246/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_246/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_246/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_246"
  op: "Mul"
  input: "Mul_246/x"
  input: "Mul_246/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_41"
  op: "Square"
  input: "CrossReplicaSum_41"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_247/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_247"
  op: "Mul"
  input: "Mul_247/x"
  input: "Square_41"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_153"
  op: "AddV2"
  input: "Mul_246"
  input: "Mul_247"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_41"
  op: "Sqrt"
  input: "add_153"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_154/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_154"
  op: "AddV2"
  input: "Sqrt_41"
  input: "add_154/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_44"
  op: "RealDiv"
  input: "add_152"
  input: "add_154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_181"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_248/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_248"
  op: "Mul"
  input: "mul_248/x"
  input: "ReadVariableOp_181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_155"
  op: "AddV2"
  input: "truediv_44"
  input: "mul_248"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_249"
  op: "Mul"
  input: "mul_21"
  input: "add_155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_182"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_45"
  op: "Sub"
  input: "ReadVariableOp_182"
  input: "mul_249"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_123"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_45"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_183"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_123"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_124"
  op: "AssignVariableOp"
  input: "Mul_244/ReadVariableOp/Enter"
  input: "add_152"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_184"
  op: "ReadVariableOp"
  input: "Mul_244/ReadVariableOp/Enter"
  input: "^AssignVariableOp_124"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_125"
  op: "AssignVariableOp"
  input: "Mul_246/ReadVariableOp/Enter"
  input: "add_153"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_185"
  op: "ReadVariableOp"
  input: "Mul_246/ReadVariableOp/Enter"
  input: "^AssignVariableOp_125"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_250/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_250/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_250/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_250/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_250"
  op: "Mul"
  input: "Mul_250/x"
  input: "Mul_250/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_251/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_251"
  op: "Mul"
  input: "Mul_251/x"
  input: "CrossReplicaSum_42"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_156"
  op: "AddV2"
  input: "Mul_250"
  input: "Mul_251"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_252/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_252/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_252/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_252/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_252"
  op: "Mul"
  input: "Mul_252/x"
  input: "Mul_252/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_42"
  op: "Square"
  input: "CrossReplicaSum_42"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_253/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_253"
  op: "Mul"
  input: "Mul_253/x"
  input: "Square_42"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_157"
  op: "AddV2"
  input: "Mul_252"
  input: "Mul_253"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_42"
  op: "Sqrt"
  input: "add_157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_158/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_158"
  op: "AddV2"
  input: "Sqrt_42"
  input: "add_158/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_45"
  op: "RealDiv"
  input: "add_156"
  input: "add_158"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_254"
  op: "Mul"
  input: "mul_21"
  input: "truediv_45"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_186"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_46"
  op: "Sub"
  input: "ReadVariableOp_186"
  input: "mul_254"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_126"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_46"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_187"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_126"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_127"
  op: "AssignVariableOp"
  input: "Mul_250/ReadVariableOp/Enter"
  input: "add_156"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_188"
  op: "ReadVariableOp"
  input: "Mul_250/ReadVariableOp/Enter"
  input: "^AssignVariableOp_127"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_128"
  op: "AssignVariableOp"
  input: "Mul_252/ReadVariableOp/Enter"
  input: "add_157"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_189"
  op: "ReadVariableOp"
  input: "Mul_252/ReadVariableOp/Enter"
  input: "^AssignVariableOp_128"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_255/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_255/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_255/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_255/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_255"
  op: "Mul"
  input: "Mul_255/x"
  input: "Mul_255/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_256/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_256"
  op: "Mul"
  input: "Mul_256/x"
  input: "CrossReplicaSum_43"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_159"
  op: "AddV2"
  input: "Mul_255"
  input: "Mul_256"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_257/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_257/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_257/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_257/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_257"
  op: "Mul"
  input: "Mul_257/x"
  input: "Mul_257/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_43"
  op: "Square"
  input: "CrossReplicaSum_43"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_258/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_258"
  op: "Mul"
  input: "Mul_258/x"
  input: "Square_43"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_160"
  op: "AddV2"
  input: "Mul_257"
  input: "Mul_258"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_43"
  op: "Sqrt"
  input: "add_160"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_161/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_161"
  op: "AddV2"
  input: "Sqrt_43"
  input: "add_161/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_46"
  op: "RealDiv"
  input: "add_159"
  input: "add_161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_190"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_259/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_259"
  op: "Mul"
  input: "mul_259/x"
  input: "ReadVariableOp_190"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_162"
  op: "AddV2"
  input: "truediv_46"
  input: "mul_259"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_260"
  op: "Mul"
  input: "mul_21"
  input: "add_162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_191"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_47"
  op: "Sub"
  input: "ReadVariableOp_191"
  input: "mul_260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_129"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_47"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_192"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_129"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_130"
  op: "AssignVariableOp"
  input: "Mul_255/ReadVariableOp/Enter"
  input: "add_159"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_193"
  op: "ReadVariableOp"
  input: "Mul_255/ReadVariableOp/Enter"
  input: "^AssignVariableOp_130"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_131"
  op: "AssignVariableOp"
  input: "Mul_257/ReadVariableOp/Enter"
  input: "add_160"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_194"
  op: "ReadVariableOp"
  input: "Mul_257/ReadVariableOp/Enter"
  input: "^AssignVariableOp_131"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_261/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_261/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_261/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_261/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_261"
  op: "Mul"
  input: "Mul_261/x"
  input: "Mul_261/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_262/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_262"
  op: "Mul"
  input: "Mul_262/x"
  input: "CrossReplicaSum_44"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_163"
  op: "AddV2"
  input: "Mul_261"
  input: "Mul_262"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_263/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_263/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_263/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_263/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_263"
  op: "Mul"
  input: "Mul_263/x"
  input: "Mul_263/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_44"
  op: "Square"
  input: "CrossReplicaSum_44"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_264/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_264"
  op: "Mul"
  input: "Mul_264/x"
  input: "Square_44"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_164"
  op: "AddV2"
  input: "Mul_263"
  input: "Mul_264"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_44"
  op: "Sqrt"
  input: "add_164"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_165/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_165"
  op: "AddV2"
  input: "Sqrt_44"
  input: "add_165/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_47"
  op: "RealDiv"
  input: "add_163"
  input: "add_165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_265"
  op: "Mul"
  input: "mul_21"
  input: "truediv_47"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_195"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_48"
  op: "Sub"
  input: "ReadVariableOp_195"
  input: "mul_265"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_132"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_48"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_196"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_132"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_133"
  op: "AssignVariableOp"
  input: "Mul_261/ReadVariableOp/Enter"
  input: "add_163"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_197"
  op: "ReadVariableOp"
  input: "Mul_261/ReadVariableOp/Enter"
  input: "^AssignVariableOp_133"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_134"
  op: "AssignVariableOp"
  input: "Mul_263/ReadVariableOp/Enter"
  input: "add_164"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_198"
  op: "ReadVariableOp"
  input: "Mul_263/ReadVariableOp/Enter"
  input: "^AssignVariableOp_134"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_266/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_266/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_266/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_266/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_266"
  op: "Mul"
  input: "Mul_266/x"
  input: "Mul_266/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_267/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_267"
  op: "Mul"
  input: "Mul_267/x"
  input: "CrossReplicaSum_45"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_166"
  op: "AddV2"
  input: "Mul_266"
  input: "Mul_267"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_268/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_268/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_268/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_268/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_268"
  op: "Mul"
  input: "Mul_268/x"
  input: "Mul_268/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_45"
  op: "Square"
  input: "CrossReplicaSum_45"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_269/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_269"
  op: "Mul"
  input: "Mul_269/x"
  input: "Square_45"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_167"
  op: "AddV2"
  input: "Mul_268"
  input: "Mul_269"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_45"
  op: "Sqrt"
  input: "add_167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_168/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_168"
  op: "AddV2"
  input: "Sqrt_45"
  input: "add_168/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_48"
  op: "RealDiv"
  input: "add_166"
  input: "add_168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_199"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_270/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_270"
  op: "Mul"
  input: "mul_270/x"
  input: "ReadVariableOp_199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_169"
  op: "AddV2"
  input: "truediv_48"
  input: "mul_270"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_271"
  op: "Mul"
  input: "mul_21"
  input: "add_169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_200"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_49"
  op: "Sub"
  input: "ReadVariableOp_200"
  input: "mul_271"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_135"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_49"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_201"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_135"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_136"
  op: "AssignVariableOp"
  input: "Mul_266/ReadVariableOp/Enter"
  input: "add_166"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_202"
  op: "ReadVariableOp"
  input: "Mul_266/ReadVariableOp/Enter"
  input: "^AssignVariableOp_136"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_137"
  op: "AssignVariableOp"
  input: "Mul_268/ReadVariableOp/Enter"
  input: "add_167"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_203"
  op: "ReadVariableOp"
  input: "Mul_268/ReadVariableOp/Enter"
  input: "^AssignVariableOp_137"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_272/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_272/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_272/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_272/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_272"
  op: "Mul"
  input: "Mul_272/x"
  input: "Mul_272/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_273/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_273"
  op: "Mul"
  input: "Mul_273/x"
  input: "CrossReplicaSum_46"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_170"
  op: "AddV2"
  input: "Mul_272"
  input: "Mul_273"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_274/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_274/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_274/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_274/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_274"
  op: "Mul"
  input: "Mul_274/x"
  input: "Mul_274/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_46"
  op: "Square"
  input: "CrossReplicaSum_46"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_275/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_275"
  op: "Mul"
  input: "Mul_275/x"
  input: "Square_46"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_171"
  op: "AddV2"
  input: "Mul_274"
  input: "Mul_275"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_46"
  op: "Sqrt"
  input: "add_171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_172/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_172"
  op: "AddV2"
  input: "Sqrt_46"
  input: "add_172/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_49"
  op: "RealDiv"
  input: "add_170"
  input: "add_172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_276"
  op: "Mul"
  input: "mul_21"
  input: "truediv_49"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_204"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_50"
  op: "Sub"
  input: "ReadVariableOp_204"
  input: "mul_276"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_138"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_50"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_205"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_138"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_139"
  op: "AssignVariableOp"
  input: "Mul_272/ReadVariableOp/Enter"
  input: "add_170"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_206"
  op: "ReadVariableOp"
  input: "Mul_272/ReadVariableOp/Enter"
  input: "^AssignVariableOp_139"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_140"
  op: "AssignVariableOp"
  input: "Mul_274/ReadVariableOp/Enter"
  input: "add_171"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_207"
  op: "ReadVariableOp"
  input: "Mul_274/ReadVariableOp/Enter"
  input: "^AssignVariableOp_140"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_277/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_277/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_277/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_277/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_277"
  op: "Mul"
  input: "Mul_277/x"
  input: "Mul_277/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_278/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_278"
  op: "Mul"
  input: "Mul_278/x"
  input: "CrossReplicaSum_47"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_173"
  op: "AddV2"
  input: "Mul_277"
  input: "Mul_278"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_279/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_279/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_279/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_279/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_279"
  op: "Mul"
  input: "Mul_279/x"
  input: "Mul_279/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_47"
  op: "Square"
  input: "CrossReplicaSum_47"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_280/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_280"
  op: "Mul"
  input: "Mul_280/x"
  input: "Square_47"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_174"
  op: "AddV2"
  input: "Mul_279"
  input: "Mul_280"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_47"
  op: "Sqrt"
  input: "add_174"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_175/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_175"
  op: "AddV2"
  input: "Sqrt_47"
  input: "add_175/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_50"
  op: "RealDiv"
  input: "add_173"
  input: "add_175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_281"
  op: "Mul"
  input: "mul_21"
  input: "truediv_50"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_208"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_51"
  op: "Sub"
  input: "ReadVariableOp_208"
  input: "mul_281"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_141"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_51"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_209"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_141"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_142"
  op: "AssignVariableOp"
  input: "Mul_277/ReadVariableOp/Enter"
  input: "add_173"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_210"
  op: "ReadVariableOp"
  input: "Mul_277/ReadVariableOp/Enter"
  input: "^AssignVariableOp_142"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_143"
  op: "AssignVariableOp"
  input: "Mul_279/ReadVariableOp/Enter"
  input: "add_174"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_211"
  op: "ReadVariableOp"
  input: "Mul_279/ReadVariableOp/Enter"
  input: "^AssignVariableOp_143"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_282/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_282/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_282/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_282/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_282"
  op: "Mul"
  input: "Mul_282/x"
  input: "Mul_282/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_283/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_283"
  op: "Mul"
  input: "Mul_283/x"
  input: "CrossReplicaSum_48"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_176"
  op: "AddV2"
  input: "Mul_282"
  input: "Mul_283"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_284/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_284/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_284/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_284/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_284"
  op: "Mul"
  input: "Mul_284/x"
  input: "Mul_284/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_48"
  op: "Square"
  input: "CrossReplicaSum_48"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_285/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_285"
  op: "Mul"
  input: "Mul_285/x"
  input: "Square_48"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_177"
  op: "AddV2"
  input: "Mul_284"
  input: "Mul_285"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_48"
  op: "Sqrt"
  input: "add_177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_178/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_178"
  op: "AddV2"
  input: "Sqrt_48"
  input: "add_178/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_51"
  op: "RealDiv"
  input: "add_176"
  input: "add_178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_286"
  op: "Mul"
  input: "mul_21"
  input: "truediv_51"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_212"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_52"
  op: "Sub"
  input: "ReadVariableOp_212"
  input: "mul_286"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_144"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_52"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_213"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_144"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_145"
  op: "AssignVariableOp"
  input: "Mul_282/ReadVariableOp/Enter"
  input: "add_176"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_214"
  op: "ReadVariableOp"
  input: "Mul_282/ReadVariableOp/Enter"
  input: "^AssignVariableOp_145"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_146"
  op: "AssignVariableOp"
  input: "Mul_284/ReadVariableOp/Enter"
  input: "add_177"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_215"
  op: "ReadVariableOp"
  input: "Mul_284/ReadVariableOp/Enter"
  input: "^AssignVariableOp_146"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_287/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_287/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_287/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_287/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_287"
  op: "Mul"
  input: "Mul_287/x"
  input: "Mul_287/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_288/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_288"
  op: "Mul"
  input: "Mul_288/x"
  input: "CrossReplicaSum_49"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_179"
  op: "AddV2"
  input: "Mul_287"
  input: "Mul_288"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_289/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_289/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_289/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_289/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_289"
  op: "Mul"
  input: "Mul_289/x"
  input: "Mul_289/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_49"
  op: "Square"
  input: "CrossReplicaSum_49"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_290/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_290"
  op: "Mul"
  input: "Mul_290/x"
  input: "Square_49"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_180"
  op: "AddV2"
  input: "Mul_289"
  input: "Mul_290"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_49"
  op: "Sqrt"
  input: "add_180"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_181/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_181"
  op: "AddV2"
  input: "Sqrt_49"
  input: "add_181/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_52"
  op: "RealDiv"
  input: "add_179"
  input: "add_181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_216"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_291/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_291"
  op: "Mul"
  input: "mul_291/x"
  input: "ReadVariableOp_216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_182"
  op: "AddV2"
  input: "truediv_52"
  input: "mul_291"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_292"
  op: "Mul"
  input: "mul_21"
  input: "add_182"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_217"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_53"
  op: "Sub"
  input: "ReadVariableOp_217"
  input: "mul_292"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_147"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_53"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_218"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_147"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_148"
  op: "AssignVariableOp"
  input: "Mul_287/ReadVariableOp/Enter"
  input: "add_179"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_219"
  op: "ReadVariableOp"
  input: "Mul_287/ReadVariableOp/Enter"
  input: "^AssignVariableOp_148"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_149"
  op: "AssignVariableOp"
  input: "Mul_289/ReadVariableOp/Enter"
  input: "add_180"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_220"
  op: "ReadVariableOp"
  input: "Mul_289/ReadVariableOp/Enter"
  input: "^AssignVariableOp_149"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_293/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_293/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_293/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_293/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_293"
  op: "Mul"
  input: "Mul_293/x"
  input: "Mul_293/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_294/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_294"
  op: "Mul"
  input: "Mul_294/x"
  input: "CrossReplicaSum_50"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_183"
  op: "AddV2"
  input: "Mul_293"
  input: "Mul_294"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_295/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_295/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_295/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_295/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_295"
  op: "Mul"
  input: "Mul_295/x"
  input: "Mul_295/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_50"
  op: "Square"
  input: "CrossReplicaSum_50"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_296/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_296"
  op: "Mul"
  input: "Mul_296/x"
  input: "Square_50"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_184"
  op: "AddV2"
  input: "Mul_295"
  input: "Mul_296"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_50"
  op: "Sqrt"
  input: "add_184"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_185/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_185"
  op: "AddV2"
  input: "Sqrt_50"
  input: "add_185/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_53"
  op: "RealDiv"
  input: "add_183"
  input: "add_185"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_297"
  op: "Mul"
  input: "mul_21"
  input: "truediv_53"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_221"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_54"
  op: "Sub"
  input: "ReadVariableOp_221"
  input: "mul_297"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_150"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_54"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_222"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_150"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_151"
  op: "AssignVariableOp"
  input: "Mul_293/ReadVariableOp/Enter"
  input: "add_183"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_223"
  op: "ReadVariableOp"
  input: "Mul_293/ReadVariableOp/Enter"
  input: "^AssignVariableOp_151"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_152"
  op: "AssignVariableOp"
  input: "Mul_295/ReadVariableOp/Enter"
  input: "add_184"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_224"
  op: "ReadVariableOp"
  input: "Mul_295/ReadVariableOp/Enter"
  input: "^AssignVariableOp_152"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_298/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_298/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_298/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_298/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_298"
  op: "Mul"
  input: "Mul_298/x"
  input: "Mul_298/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_299/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_299"
  op: "Mul"
  input: "Mul_299/x"
  input: "CrossReplicaSum_51"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_186"
  op: "AddV2"
  input: "Mul_298"
  input: "Mul_299"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_300/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_300/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_300/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_300/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_300"
  op: "Mul"
  input: "Mul_300/x"
  input: "Mul_300/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_51"
  op: "Square"
  input: "CrossReplicaSum_51"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_301/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_301"
  op: "Mul"
  input: "Mul_301/x"
  input: "Square_51"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_187"
  op: "AddV2"
  input: "Mul_300"
  input: "Mul_301"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_51"
  op: "Sqrt"
  input: "add_187"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_188/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_188"
  op: "AddV2"
  input: "Sqrt_51"
  input: "add_188/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_54"
  op: "RealDiv"
  input: "add_186"
  input: "add_188"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_225"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_302/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_302"
  op: "Mul"
  input: "mul_302/x"
  input: "ReadVariableOp_225"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_189"
  op: "AddV2"
  input: "truediv_54"
  input: "mul_302"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_303"
  op: "Mul"
  input: "mul_21"
  input: "add_189"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_226"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_55"
  op: "Sub"
  input: "ReadVariableOp_226"
  input: "mul_303"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_153"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_55"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_227"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_153"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_154"
  op: "AssignVariableOp"
  input: "Mul_298/ReadVariableOp/Enter"
  input: "add_186"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_228"
  op: "ReadVariableOp"
  input: "Mul_298/ReadVariableOp/Enter"
  input: "^AssignVariableOp_154"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_155"
  op: "AssignVariableOp"
  input: "Mul_300/ReadVariableOp/Enter"
  input: "add_187"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_229"
  op: "ReadVariableOp"
  input: "Mul_300/ReadVariableOp/Enter"
  input: "^AssignVariableOp_155"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_m"
  input: "generator/encoder/layer_2/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_v"
  input: "generator/encoder/layer_2/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_304/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_304/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_304/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_304/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_304"
  op: "Mul"
  input: "Mul_304/x"
  input: "Mul_304/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_305/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_305"
  op: "Mul"
  input: "Mul_305/x"
  input: "CrossReplicaSum_52"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_190"
  op: "AddV2"
  input: "Mul_304"
  input: "Mul_305"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_306/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_306/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_306/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_306/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_306"
  op: "Mul"
  input: "Mul_306/x"
  input: "Mul_306/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_52"
  op: "Square"
  input: "CrossReplicaSum_52"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_307/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_307"
  op: "Mul"
  input: "Mul_307/x"
  input: "Square_52"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_191"
  op: "AddV2"
  input: "Mul_306"
  input: "Mul_307"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_52"
  op: "Sqrt"
  input: "add_191"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_192/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_192"
  op: "AddV2"
  input: "Sqrt_52"
  input: "add_192/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_55"
  op: "RealDiv"
  input: "add_190"
  input: "add_192"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_308"
  op: "Mul"
  input: "mul_21"
  input: "truediv_55"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_230"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_56"
  op: "Sub"
  input: "ReadVariableOp_230"
  input: "mul_308"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_156"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_56"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_231"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_156"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_157"
  op: "AssignVariableOp"
  input: "Mul_304/ReadVariableOp/Enter"
  input: "add_190"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_232"
  op: "ReadVariableOp"
  input: "Mul_304/ReadVariableOp/Enter"
  input: "^AssignVariableOp_157"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_158"
  op: "AssignVariableOp"
  input: "Mul_306/ReadVariableOp/Enter"
  input: "add_191"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_233"
  op: "ReadVariableOp"
  input: "Mul_306/ReadVariableOp/Enter"
  input: "^AssignVariableOp_158"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_309/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_309/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_309/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_309/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_309"
  op: "Mul"
  input: "Mul_309/x"
  input: "Mul_309/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_310/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_310"
  op: "Mul"
  input: "Mul_310/x"
  input: "CrossReplicaSum_53"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_193"
  op: "AddV2"
  input: "Mul_309"
  input: "Mul_310"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_311/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_311/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_311/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_311/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_311"
  op: "Mul"
  input: "Mul_311/x"
  input: "Mul_311/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_53"
  op: "Square"
  input: "CrossReplicaSum_53"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_312/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_312"
  op: "Mul"
  input: "Mul_312/x"
  input: "Square_53"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_194"
  op: "AddV2"
  input: "Mul_311"
  input: "Mul_312"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_53"
  op: "Sqrt"
  input: "add_194"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_195/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_195"
  op: "AddV2"
  input: "Sqrt_53"
  input: "add_195/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_56"
  op: "RealDiv"
  input: "add_193"
  input: "add_195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_313"
  op: "Mul"
  input: "mul_21"
  input: "truediv_56"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_234"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_57"
  op: "Sub"
  input: "ReadVariableOp_234"
  input: "mul_313"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_159"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_57"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_235"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_159"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_160"
  op: "AssignVariableOp"
  input: "Mul_309/ReadVariableOp/Enter"
  input: "add_193"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_236"
  op: "ReadVariableOp"
  input: "Mul_309/ReadVariableOp/Enter"
  input: "^AssignVariableOp_160"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_161"
  op: "AssignVariableOp"
  input: "Mul_311/ReadVariableOp/Enter"
  input: "add_194"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_237"
  op: "ReadVariableOp"
  input: "Mul_311/ReadVariableOp/Enter"
  input: "^AssignVariableOp_161"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_314/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_314/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_314/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_314/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_314"
  op: "Mul"
  input: "Mul_314/x"
  input: "Mul_314/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_315/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_315"
  op: "Mul"
  input: "Mul_315/x"
  input: "CrossReplicaSum_54"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_196"
  op: "AddV2"
  input: "Mul_314"
  input: "Mul_315"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_316/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_316/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_316/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_316/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_316"
  op: "Mul"
  input: "Mul_316/x"
  input: "Mul_316/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_54"
  op: "Square"
  input: "CrossReplicaSum_54"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_317/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_317"
  op: "Mul"
  input: "Mul_317/x"
  input: "Square_54"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_197"
  op: "AddV2"
  input: "Mul_316"
  input: "Mul_317"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_54"
  op: "Sqrt"
  input: "add_197"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_198/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_198"
  op: "AddV2"
  input: "Sqrt_54"
  input: "add_198/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_57"
  op: "RealDiv"
  input: "add_196"
  input: "add_198"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_318"
  op: "Mul"
  input: "mul_21"
  input: "truediv_57"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_238"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_58"
  op: "Sub"
  input: "ReadVariableOp_238"
  input: "mul_318"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_162"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_58"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_239"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_162"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_163"
  op: "AssignVariableOp"
  input: "Mul_314/ReadVariableOp/Enter"
  input: "add_196"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_240"
  op: "ReadVariableOp"
  input: "Mul_314/ReadVariableOp/Enter"
  input: "^AssignVariableOp_163"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_164"
  op: "AssignVariableOp"
  input: "Mul_316/ReadVariableOp/Enter"
  input: "add_197"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_241"
  op: "ReadVariableOp"
  input: "Mul_316/ReadVariableOp/Enter"
  input: "^AssignVariableOp_164"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_319/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_319/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_319/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_319/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_319"
  op: "Mul"
  input: "Mul_319/x"
  input: "Mul_319/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_320/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_320"
  op: "Mul"
  input: "Mul_320/x"
  input: "CrossReplicaSum_55"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_199"
  op: "AddV2"
  input: "Mul_319"
  input: "Mul_320"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_321/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_321/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_321/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_321/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_321"
  op: "Mul"
  input: "Mul_321/x"
  input: "Mul_321/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_55"
  op: "Square"
  input: "CrossReplicaSum_55"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_322/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_322"
  op: "Mul"
  input: "Mul_322/x"
  input: "Square_55"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_200"
  op: "AddV2"
  input: "Mul_321"
  input: "Mul_322"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_55"
  op: "Sqrt"
  input: "add_200"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_201/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_201"
  op: "AddV2"
  input: "Sqrt_55"
  input: "add_201/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_58"
  op: "RealDiv"
  input: "add_199"
  input: "add_201"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_242"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_323/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_323"
  op: "Mul"
  input: "mul_323/x"
  input: "ReadVariableOp_242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_202"
  op: "AddV2"
  input: "truediv_58"
  input: "mul_323"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_324"
  op: "Mul"
  input: "mul_21"
  input: "add_202"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_243"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_59"
  op: "Sub"
  input: "ReadVariableOp_243"
  input: "mul_324"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_165"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_59"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_244"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_165"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_166"
  op: "AssignVariableOp"
  input: "Mul_319/ReadVariableOp/Enter"
  input: "add_199"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_245"
  op: "ReadVariableOp"
  input: "Mul_319/ReadVariableOp/Enter"
  input: "^AssignVariableOp_166"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_167"
  op: "AssignVariableOp"
  input: "Mul_321/ReadVariableOp/Enter"
  input: "add_200"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_246"
  op: "ReadVariableOp"
  input: "Mul_321/ReadVariableOp/Enter"
  input: "^AssignVariableOp_167"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_325/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_325/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_325/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_325/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_325"
  op: "Mul"
  input: "Mul_325/x"
  input: "Mul_325/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_326/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_326"
  op: "Mul"
  input: "Mul_326/x"
  input: "CrossReplicaSum_56"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_203"
  op: "AddV2"
  input: "Mul_325"
  input: "Mul_326"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_327/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_327/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_327/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_327/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_327"
  op: "Mul"
  input: "Mul_327/x"
  input: "Mul_327/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_56"
  op: "Square"
  input: "CrossReplicaSum_56"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_328/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_328"
  op: "Mul"
  input: "Mul_328/x"
  input: "Square_56"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_204"
  op: "AddV2"
  input: "Mul_327"
  input: "Mul_328"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_56"
  op: "Sqrt"
  input: "add_204"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_205/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_205"
  op: "AddV2"
  input: "Sqrt_56"
  input: "add_205/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_59"
  op: "RealDiv"
  input: "add_203"
  input: "add_205"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_329"
  op: "Mul"
  input: "mul_21"
  input: "truediv_59"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_247"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_60"
  op: "Sub"
  input: "ReadVariableOp_247"
  input: "mul_329"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_168"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_60"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_248"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_168"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_169"
  op: "AssignVariableOp"
  input: "Mul_325/ReadVariableOp/Enter"
  input: "add_203"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_249"
  op: "ReadVariableOp"
  input: "Mul_325/ReadVariableOp/Enter"
  input: "^AssignVariableOp_169"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_170"
  op: "AssignVariableOp"
  input: "Mul_327/ReadVariableOp/Enter"
  input: "add_204"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_250"
  op: "ReadVariableOp"
  input: "Mul_327/ReadVariableOp/Enter"
  input: "^AssignVariableOp_170"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_330/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_330/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_330/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_330/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_330"
  op: "Mul"
  input: "Mul_330/x"
  input: "Mul_330/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_331/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_331"
  op: "Mul"
  input: "Mul_331/x"
  input: "CrossReplicaSum_57"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_206"
  op: "AddV2"
  input: "Mul_330"
  input: "Mul_331"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_332/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_332/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_332/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_332/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_332"
  op: "Mul"
  input: "Mul_332/x"
  input: "Mul_332/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_57"
  op: "Square"
  input: "CrossReplicaSum_57"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_333/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_333"
  op: "Mul"
  input: "Mul_333/x"
  input: "Square_57"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_207"
  op: "AddV2"
  input: "Mul_332"
  input: "Mul_333"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_57"
  op: "Sqrt"
  input: "add_207"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_208/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_208"
  op: "AddV2"
  input: "Sqrt_57"
  input: "add_208/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_60"
  op: "RealDiv"
  input: "add_206"
  input: "add_208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_251"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_334/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_334"
  op: "Mul"
  input: "mul_334/x"
  input: "ReadVariableOp_251"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_209"
  op: "AddV2"
  input: "truediv_60"
  input: "mul_334"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_335"
  op: "Mul"
  input: "mul_21"
  input: "add_209"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_252"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_61"
  op: "Sub"
  input: "ReadVariableOp_252"
  input: "mul_335"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_171"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_61"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_253"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_171"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_172"
  op: "AssignVariableOp"
  input: "Mul_330/ReadVariableOp/Enter"
  input: "add_206"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_254"
  op: "ReadVariableOp"
  input: "Mul_330/ReadVariableOp/Enter"
  input: "^AssignVariableOp_172"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_173"
  op: "AssignVariableOp"
  input: "Mul_332/ReadVariableOp/Enter"
  input: "add_207"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_255"
  op: "ReadVariableOp"
  input: "Mul_332/ReadVariableOp/Enter"
  input: "^AssignVariableOp_173"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_336/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_336/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_336/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_336/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_336"
  op: "Mul"
  input: "Mul_336/x"
  input: "Mul_336/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_337/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_337"
  op: "Mul"
  input: "Mul_337/x"
  input: "CrossReplicaSum_58"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_210"
  op: "AddV2"
  input: "Mul_336"
  input: "Mul_337"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_338/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_338/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_338/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_338/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_338"
  op: "Mul"
  input: "Mul_338/x"
  input: "Mul_338/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_58"
  op: "Square"
  input: "CrossReplicaSum_58"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_339/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_339"
  op: "Mul"
  input: "Mul_339/x"
  input: "Square_58"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_211"
  op: "AddV2"
  input: "Mul_338"
  input: "Mul_339"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_58"
  op: "Sqrt"
  input: "add_211"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_212/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_212"
  op: "AddV2"
  input: "Sqrt_58"
  input: "add_212/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_61"
  op: "RealDiv"
  input: "add_210"
  input: "add_212"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_340"
  op: "Mul"
  input: "mul_21"
  input: "truediv_61"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_256"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_62"
  op: "Sub"
  input: "ReadVariableOp_256"
  input: "mul_340"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_174"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_62"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_257"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_174"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_175"
  op: "AssignVariableOp"
  input: "Mul_336/ReadVariableOp/Enter"
  input: "add_210"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_258"
  op: "ReadVariableOp"
  input: "Mul_336/ReadVariableOp/Enter"
  input: "^AssignVariableOp_175"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_176"
  op: "AssignVariableOp"
  input: "Mul_338/ReadVariableOp/Enter"
  input: "add_211"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_259"
  op: "ReadVariableOp"
  input: "Mul_338/ReadVariableOp/Enter"
  input: "^AssignVariableOp_176"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_341/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_341/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_341/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_341/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_341"
  op: "Mul"
  input: "Mul_341/x"
  input: "Mul_341/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_342/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_342"
  op: "Mul"
  input: "Mul_342/x"
  input: "CrossReplicaSum_59"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_213"
  op: "AddV2"
  input: "Mul_341"
  input: "Mul_342"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_343/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_343/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_343/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_343/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_343"
  op: "Mul"
  input: "Mul_343/x"
  input: "Mul_343/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_59"
  op: "Square"
  input: "CrossReplicaSum_59"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_344/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_344"
  op: "Mul"
  input: "Mul_344/x"
  input: "Square_59"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_214"
  op: "AddV2"
  input: "Mul_343"
  input: "Mul_344"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_59"
  op: "Sqrt"
  input: "add_214"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_215/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_215"
  op: "AddV2"
  input: "Sqrt_59"
  input: "add_215/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_62"
  op: "RealDiv"
  input: "add_213"
  input: "add_215"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_260"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_345/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_345"
  op: "Mul"
  input: "mul_345/x"
  input: "ReadVariableOp_260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_216"
  op: "AddV2"
  input: "truediv_62"
  input: "mul_345"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_346"
  op: "Mul"
  input: "mul_21"
  input: "add_216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_261"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_63"
  op: "Sub"
  input: "ReadVariableOp_261"
  input: "mul_346"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_177"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_63"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_262"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_177"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_178"
  op: "AssignVariableOp"
  input: "Mul_341/ReadVariableOp/Enter"
  input: "add_213"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_263"
  op: "ReadVariableOp"
  input: "Mul_341/ReadVariableOp/Enter"
  input: "^AssignVariableOp_178"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_179"
  op: "AssignVariableOp"
  input: "Mul_343/ReadVariableOp/Enter"
  input: "add_214"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_264"
  op: "ReadVariableOp"
  input: "Mul_343/ReadVariableOp/Enter"
  input: "^AssignVariableOp_179"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_347/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_347/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_347/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_347/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_347"
  op: "Mul"
  input: "Mul_347/x"
  input: "Mul_347/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_348/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_348"
  op: "Mul"
  input: "Mul_348/x"
  input: "CrossReplicaSum_60"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_217"
  op: "AddV2"
  input: "Mul_347"
  input: "Mul_348"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_349/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_349/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_349/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_349/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_349"
  op: "Mul"
  input: "Mul_349/x"
  input: "Mul_349/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_60"
  op: "Square"
  input: "CrossReplicaSum_60"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_350/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_350"
  op: "Mul"
  input: "Mul_350/x"
  input: "Square_60"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_218"
  op: "AddV2"
  input: "Mul_349"
  input: "Mul_350"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_60"
  op: "Sqrt"
  input: "add_218"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_219/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_219"
  op: "AddV2"
  input: "Sqrt_60"
  input: "add_219/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_63"
  op: "RealDiv"
  input: "add_217"
  input: "add_219"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_351"
  op: "Mul"
  input: "mul_21"
  input: "truediv_63"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_265"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_64"
  op: "Sub"
  input: "ReadVariableOp_265"
  input: "mul_351"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_180"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_64"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_266"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_180"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_181"
  op: "AssignVariableOp"
  input: "Mul_347/ReadVariableOp/Enter"
  input: "add_217"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_267"
  op: "ReadVariableOp"
  input: "Mul_347/ReadVariableOp/Enter"
  input: "^AssignVariableOp_181"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_182"
  op: "AssignVariableOp"
  input: "Mul_349/ReadVariableOp/Enter"
  input: "add_218"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_268"
  op: "ReadVariableOp"
  input: "Mul_349/ReadVariableOp/Enter"
  input: "^AssignVariableOp_182"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_352/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_352/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_352/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_352/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_352"
  op: "Mul"
  input: "Mul_352/x"
  input: "Mul_352/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_353/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_353"
  op: "Mul"
  input: "Mul_353/x"
  input: "CrossReplicaSum_61"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_220"
  op: "AddV2"
  input: "Mul_352"
  input: "Mul_353"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_354/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_354/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_354/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_354/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_354"
  op: "Mul"
  input: "Mul_354/x"
  input: "Mul_354/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_61"
  op: "Square"
  input: "CrossReplicaSum_61"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_355/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_355"
  op: "Mul"
  input: "Mul_355/x"
  input: "Square_61"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_221"
  op: "AddV2"
  input: "Mul_354"
  input: "Mul_355"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_61"
  op: "Sqrt"
  input: "add_221"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_222/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_222"
  op: "AddV2"
  input: "Sqrt_61"
  input: "add_222/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_64"
  op: "RealDiv"
  input: "add_220"
  input: "add_222"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_269"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_356/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_356"
  op: "Mul"
  input: "mul_356/x"
  input: "ReadVariableOp_269"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_223"
  op: "AddV2"
  input: "truediv_64"
  input: "mul_356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_357"
  op: "Mul"
  input: "mul_21"
  input: "add_223"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_270"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_65"
  op: "Sub"
  input: "ReadVariableOp_270"
  input: "mul_357"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_183"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_65"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_271"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_183"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_184"
  op: "AssignVariableOp"
  input: "Mul_352/ReadVariableOp/Enter"
  input: "add_220"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_272"
  op: "ReadVariableOp"
  input: "Mul_352/ReadVariableOp/Enter"
  input: "^AssignVariableOp_184"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_185"
  op: "AssignVariableOp"
  input: "Mul_354/ReadVariableOp/Enter"
  input: "add_221"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_273"
  op: "ReadVariableOp"
  input: "Mul_354/ReadVariableOp/Enter"
  input: "^AssignVariableOp_185"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_358/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_358/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_358/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_358/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_358"
  op: "Mul"
  input: "Mul_358/x"
  input: "Mul_358/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_359/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_359"
  op: "Mul"
  input: "Mul_359/x"
  input: "CrossReplicaSum_62"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_224"
  op: "AddV2"
  input: "Mul_358"
  input: "Mul_359"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_360/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_360/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_360/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_360/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_360"
  op: "Mul"
  input: "Mul_360/x"
  input: "Mul_360/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_62"
  op: "Square"
  input: "CrossReplicaSum_62"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_361/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_361"
  op: "Mul"
  input: "Mul_361/x"
  input: "Square_62"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_225"
  op: "AddV2"
  input: "Mul_360"
  input: "Mul_361"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_62"
  op: "Sqrt"
  input: "add_225"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_226/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_226"
  op: "AddV2"
  input: "Sqrt_62"
  input: "add_226/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_65"
  op: "RealDiv"
  input: "add_224"
  input: "add_226"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_362"
  op: "Mul"
  input: "mul_21"
  input: "truediv_65"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_274"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_66"
  op: "Sub"
  input: "ReadVariableOp_274"
  input: "mul_362"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_186"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_66"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_275"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_186"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_187"
  op: "AssignVariableOp"
  input: "Mul_358/ReadVariableOp/Enter"
  input: "add_224"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_276"
  op: "ReadVariableOp"
  input: "Mul_358/ReadVariableOp/Enter"
  input: "^AssignVariableOp_187"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_188"
  op: "AssignVariableOp"
  input: "Mul_360/ReadVariableOp/Enter"
  input: "add_225"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_277"
  op: "ReadVariableOp"
  input: "Mul_360/ReadVariableOp/Enter"
  input: "^AssignVariableOp_188"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_363/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_363/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_363/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_363/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_363"
  op: "Mul"
  input: "Mul_363/x"
  input: "Mul_363/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_364/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_364"
  op: "Mul"
  input: "Mul_364/x"
  input: "CrossReplicaSum_63"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_227"
  op: "AddV2"
  input: "Mul_363"
  input: "Mul_364"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_365/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_365/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_365/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_365/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_365"
  op: "Mul"
  input: "Mul_365/x"
  input: "Mul_365/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_63"
  op: "Square"
  input: "CrossReplicaSum_63"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_366/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_366"
  op: "Mul"
  input: "Mul_366/x"
  input: "Square_63"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_228"
  op: "AddV2"
  input: "Mul_365"
  input: "Mul_366"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_63"
  op: "Sqrt"
  input: "add_228"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_229/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_229"
  op: "AddV2"
  input: "Sqrt_63"
  input: "add_229/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_66"
  op: "RealDiv"
  input: "add_227"
  input: "add_229"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_367"
  op: "Mul"
  input: "mul_21"
  input: "truediv_66"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_278"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_67"
  op: "Sub"
  input: "ReadVariableOp_278"
  input: "mul_367"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_189"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_67"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_279"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_189"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_190"
  op: "AssignVariableOp"
  input: "Mul_363/ReadVariableOp/Enter"
  input: "add_227"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_280"
  op: "ReadVariableOp"
  input: "Mul_363/ReadVariableOp/Enter"
  input: "^AssignVariableOp_190"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_191"
  op: "AssignVariableOp"
  input: "Mul_365/ReadVariableOp/Enter"
  input: "add_228"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_281"
  op: "ReadVariableOp"
  input: "Mul_365/ReadVariableOp/Enter"
  input: "^AssignVariableOp_191"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_368/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_368/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_368/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_368/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_368"
  op: "Mul"
  input: "Mul_368/x"
  input: "Mul_368/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_369/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_369"
  op: "Mul"
  input: "Mul_369/x"
  input: "CrossReplicaSum_64"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_230"
  op: "AddV2"
  input: "Mul_368"
  input: "Mul_369"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_370/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_370/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_370/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_370/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_370"
  op: "Mul"
  input: "Mul_370/x"
  input: "Mul_370/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_64"
  op: "Square"
  input: "CrossReplicaSum_64"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_371/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_371"
  op: "Mul"
  input: "Mul_371/x"
  input: "Square_64"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_231"
  op: "AddV2"
  input: "Mul_370"
  input: "Mul_371"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_64"
  op: "Sqrt"
  input: "add_231"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_232/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_232"
  op: "AddV2"
  input: "Sqrt_64"
  input: "add_232/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_67"
  op: "RealDiv"
  input: "add_230"
  input: "add_232"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_372"
  op: "Mul"
  input: "mul_21"
  input: "truediv_67"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_282"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_68"
  op: "Sub"
  input: "ReadVariableOp_282"
  input: "mul_372"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_192"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_68"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_283"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_192"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_193"
  op: "AssignVariableOp"
  input: "Mul_368/ReadVariableOp/Enter"
  input: "add_230"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_284"
  op: "ReadVariableOp"
  input: "Mul_368/ReadVariableOp/Enter"
  input: "^AssignVariableOp_193"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_194"
  op: "AssignVariableOp"
  input: "Mul_370/ReadVariableOp/Enter"
  input: "add_231"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_285"
  op: "ReadVariableOp"
  input: "Mul_370/ReadVariableOp/Enter"
  input: "^AssignVariableOp_194"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_373/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_373/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_373/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_373/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_373"
  op: "Mul"
  input: "Mul_373/x"
  input: "Mul_373/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_374/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_374"
  op: "Mul"
  input: "Mul_374/x"
  input: "CrossReplicaSum_65"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_233"
  op: "AddV2"
  input: "Mul_373"
  input: "Mul_374"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_375/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_375/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_375/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_375/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_375"
  op: "Mul"
  input: "Mul_375/x"
  input: "Mul_375/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_65"
  op: "Square"
  input: "CrossReplicaSum_65"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_376/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_376"
  op: "Mul"
  input: "Mul_376/x"
  input: "Square_65"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_234"
  op: "AddV2"
  input: "Mul_375"
  input: "Mul_376"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_65"
  op: "Sqrt"
  input: "add_234"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_235/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_235"
  op: "AddV2"
  input: "Sqrt_65"
  input: "add_235/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_68"
  op: "RealDiv"
  input: "add_233"
  input: "add_235"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_286"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_377/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_377"
  op: "Mul"
  input: "mul_377/x"
  input: "ReadVariableOp_286"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_236"
  op: "AddV2"
  input: "truediv_68"
  input: "mul_377"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_378"
  op: "Mul"
  input: "mul_21"
  input: "add_236"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_287"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_69"
  op: "Sub"
  input: "ReadVariableOp_287"
  input: "mul_378"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_195"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_69"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_288"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_195"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_196"
  op: "AssignVariableOp"
  input: "Mul_373/ReadVariableOp/Enter"
  input: "add_233"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_289"
  op: "ReadVariableOp"
  input: "Mul_373/ReadVariableOp/Enter"
  input: "^AssignVariableOp_196"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_197"
  op: "AssignVariableOp"
  input: "Mul_375/ReadVariableOp/Enter"
  input: "add_234"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_290"
  op: "ReadVariableOp"
  input: "Mul_375/ReadVariableOp/Enter"
  input: "^AssignVariableOp_197"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_379/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_379/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_379/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_379/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_379"
  op: "Mul"
  input: "Mul_379/x"
  input: "Mul_379/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_380/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_380"
  op: "Mul"
  input: "Mul_380/x"
  input: "CrossReplicaSum_66"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_237"
  op: "AddV2"
  input: "Mul_379"
  input: "Mul_380"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_381/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_381/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_381/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_381/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_381"
  op: "Mul"
  input: "Mul_381/x"
  input: "Mul_381/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_66"
  op: "Square"
  input: "CrossReplicaSum_66"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_382/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_382"
  op: "Mul"
  input: "Mul_382/x"
  input: "Square_66"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_238"
  op: "AddV2"
  input: "Mul_381"
  input: "Mul_382"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_66"
  op: "Sqrt"
  input: "add_238"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_239/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_239"
  op: "AddV2"
  input: "Sqrt_66"
  input: "add_239/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_69"
  op: "RealDiv"
  input: "add_237"
  input: "add_239"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_383"
  op: "Mul"
  input: "mul_21"
  input: "truediv_69"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_291"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_70"
  op: "Sub"
  input: "ReadVariableOp_291"
  input: "mul_383"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_198"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_70"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_292"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_198"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_199"
  op: "AssignVariableOp"
  input: "Mul_379/ReadVariableOp/Enter"
  input: "add_237"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_293"
  op: "ReadVariableOp"
  input: "Mul_379/ReadVariableOp/Enter"
  input: "^AssignVariableOp_199"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_200"
  op: "AssignVariableOp"
  input: "Mul_381/ReadVariableOp/Enter"
  input: "add_238"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_294"
  op: "ReadVariableOp"
  input: "Mul_381/ReadVariableOp/Enter"
  input: "^AssignVariableOp_200"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_384/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_384/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_384/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_384/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_384"
  op: "Mul"
  input: "Mul_384/x"
  input: "Mul_384/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_385/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_385"
  op: "Mul"
  input: "Mul_385/x"
  input: "CrossReplicaSum_67"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_240"
  op: "AddV2"
  input: "Mul_384"
  input: "Mul_385"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_386/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_386/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_386/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_386/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_386"
  op: "Mul"
  input: "Mul_386/x"
  input: "Mul_386/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_67"
  op: "Square"
  input: "CrossReplicaSum_67"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_387/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_387"
  op: "Mul"
  input: "Mul_387/x"
  input: "Square_67"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_241"
  op: "AddV2"
  input: "Mul_386"
  input: "Mul_387"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_67"
  op: "Sqrt"
  input: "add_241"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_242/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_242"
  op: "AddV2"
  input: "Sqrt_67"
  input: "add_242/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_70"
  op: "RealDiv"
  input: "add_240"
  input: "add_242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_295"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_388/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_388"
  op: "Mul"
  input: "mul_388/x"
  input: "ReadVariableOp_295"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_243"
  op: "AddV2"
  input: "truediv_70"
  input: "mul_388"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_389"
  op: "Mul"
  input: "mul_21"
  input: "add_243"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_296"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_71"
  op: "Sub"
  input: "ReadVariableOp_296"
  input: "mul_389"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_201"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_71"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_297"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_201"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_202"
  op: "AssignVariableOp"
  input: "Mul_384/ReadVariableOp/Enter"
  input: "add_240"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_298"
  op: "ReadVariableOp"
  input: "Mul_384/ReadVariableOp/Enter"
  input: "^AssignVariableOp_202"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_203"
  op: "AssignVariableOp"
  input: "Mul_386/ReadVariableOp/Enter"
  input: "add_241"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_299"
  op: "ReadVariableOp"
  input: "Mul_386/ReadVariableOp/Enter"
  input: "^AssignVariableOp_203"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_m"
  input: "generator/encoder/layer_3/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_v"
  input: "generator/encoder/layer_3/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_390/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_390/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_390/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_390/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_390"
  op: "Mul"
  input: "Mul_390/x"
  input: "Mul_390/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_391/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_391"
  op: "Mul"
  input: "Mul_391/x"
  input: "CrossReplicaSum_68"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_244"
  op: "AddV2"
  input: "Mul_390"
  input: "Mul_391"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_392/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_392/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_392/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_392/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_392"
  op: "Mul"
  input: "Mul_392/x"
  input: "Mul_392/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_68"
  op: "Square"
  input: "CrossReplicaSum_68"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_393/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_393"
  op: "Mul"
  input: "Mul_393/x"
  input: "Square_68"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_245"
  op: "AddV2"
  input: "Mul_392"
  input: "Mul_393"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_68"
  op: "Sqrt"
  input: "add_245"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_246/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_246"
  op: "AddV2"
  input: "Sqrt_68"
  input: "add_246/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_71"
  op: "RealDiv"
  input: "add_244"
  input: "add_246"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_394"
  op: "Mul"
  input: "mul_21"
  input: "truediv_71"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_300"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_72"
  op: "Sub"
  input: "ReadVariableOp_300"
  input: "mul_394"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_204"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_72"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_301"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_204"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_205"
  op: "AssignVariableOp"
  input: "Mul_390/ReadVariableOp/Enter"
  input: "add_244"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_302"
  op: "ReadVariableOp"
  input: "Mul_390/ReadVariableOp/Enter"
  input: "^AssignVariableOp_205"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_206"
  op: "AssignVariableOp"
  input: "Mul_392/ReadVariableOp/Enter"
  input: "add_245"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_303"
  op: "ReadVariableOp"
  input: "Mul_392/ReadVariableOp/Enter"
  input: "^AssignVariableOp_206"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_395/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_395/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_395/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_395/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_395"
  op: "Mul"
  input: "Mul_395/x"
  input: "Mul_395/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_396/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_396"
  op: "Mul"
  input: "Mul_396/x"
  input: "CrossReplicaSum_69"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_247"
  op: "AddV2"
  input: "Mul_395"
  input: "Mul_396"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_397/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_397/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_397/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_397/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_397"
  op: "Mul"
  input: "Mul_397/x"
  input: "Mul_397/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_69"
  op: "Square"
  input: "CrossReplicaSum_69"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_398/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_398"
  op: "Mul"
  input: "Mul_398/x"
  input: "Square_69"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_248"
  op: "AddV2"
  input: "Mul_397"
  input: "Mul_398"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_69"
  op: "Sqrt"
  input: "add_248"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_249/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_249"
  op: "AddV2"
  input: "Sqrt_69"
  input: "add_249/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_72"
  op: "RealDiv"
  input: "add_247"
  input: "add_249"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_399"
  op: "Mul"
  input: "mul_21"
  input: "truediv_72"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_304"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_73"
  op: "Sub"
  input: "ReadVariableOp_304"
  input: "mul_399"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_207"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_73"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_305"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_207"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_208"
  op: "AssignVariableOp"
  input: "Mul_395/ReadVariableOp/Enter"
  input: "add_247"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_306"
  op: "ReadVariableOp"
  input: "Mul_395/ReadVariableOp/Enter"
  input: "^AssignVariableOp_208"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_209"
  op: "AssignVariableOp"
  input: "Mul_397/ReadVariableOp/Enter"
  input: "add_248"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_307"
  op: "ReadVariableOp"
  input: "Mul_397/ReadVariableOp/Enter"
  input: "^AssignVariableOp_209"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_400/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_400/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_400/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_400/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_400"
  op: "Mul"
  input: "Mul_400/x"
  input: "Mul_400/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_401/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_401"
  op: "Mul"
  input: "Mul_401/x"
  input: "CrossReplicaSum_70"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_250"
  op: "AddV2"
  input: "Mul_400"
  input: "Mul_401"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_402/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_402/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_402/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_402/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_402"
  op: "Mul"
  input: "Mul_402/x"
  input: "Mul_402/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_70"
  op: "Square"
  input: "CrossReplicaSum_70"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_403/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_403"
  op: "Mul"
  input: "Mul_403/x"
  input: "Square_70"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_251"
  op: "AddV2"
  input: "Mul_402"
  input: "Mul_403"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_70"
  op: "Sqrt"
  input: "add_251"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_252/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_252"
  op: "AddV2"
  input: "Sqrt_70"
  input: "add_252/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_73"
  op: "RealDiv"
  input: "add_250"
  input: "add_252"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_404"
  op: "Mul"
  input: "mul_21"
  input: "truediv_73"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_308"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_74"
  op: "Sub"
  input: "ReadVariableOp_308"
  input: "mul_404"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_210"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_74"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_309"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_210"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_211"
  op: "AssignVariableOp"
  input: "Mul_400/ReadVariableOp/Enter"
  input: "add_250"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_310"
  op: "ReadVariableOp"
  input: "Mul_400/ReadVariableOp/Enter"
  input: "^AssignVariableOp_211"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_212"
  op: "AssignVariableOp"
  input: "Mul_402/ReadVariableOp/Enter"
  input: "add_251"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_311"
  op: "ReadVariableOp"
  input: "Mul_402/ReadVariableOp/Enter"
  input: "^AssignVariableOp_212"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_405/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_405/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_405/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_405/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_405"
  op: "Mul"
  input: "Mul_405/x"
  input: "Mul_405/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_406/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_406"
  op: "Mul"
  input: "Mul_406/x"
  input: "CrossReplicaSum_71"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_253"
  op: "AddV2"
  input: "Mul_405"
  input: "Mul_406"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_407/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_407/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_407/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_407/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_407"
  op: "Mul"
  input: "Mul_407/x"
  input: "Mul_407/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_71"
  op: "Square"
  input: "CrossReplicaSum_71"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_408/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_408"
  op: "Mul"
  input: "Mul_408/x"
  input: "Square_71"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_254"
  op: "AddV2"
  input: "Mul_407"
  input: "Mul_408"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_71"
  op: "Sqrt"
  input: "add_254"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_255/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_255"
  op: "AddV2"
  input: "Sqrt_71"
  input: "add_255/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_74"
  op: "RealDiv"
  input: "add_253"
  input: "add_255"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_312"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_409/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_409"
  op: "Mul"
  input: "mul_409/x"
  input: "ReadVariableOp_312"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_256"
  op: "AddV2"
  input: "truediv_74"
  input: "mul_409"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_410"
  op: "Mul"
  input: "mul_21"
  input: "add_256"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_313"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_75"
  op: "Sub"
  input: "ReadVariableOp_313"
  input: "mul_410"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_213"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_75"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_314"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_213"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_214"
  op: "AssignVariableOp"
  input: "Mul_405/ReadVariableOp/Enter"
  input: "add_253"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_315"
  op: "ReadVariableOp"
  input: "Mul_405/ReadVariableOp/Enter"
  input: "^AssignVariableOp_214"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_215"
  op: "AssignVariableOp"
  input: "Mul_407/ReadVariableOp/Enter"
  input: "add_254"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_316"
  op: "ReadVariableOp"
  input: "Mul_407/ReadVariableOp/Enter"
  input: "^AssignVariableOp_215"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_411/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_411/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_411/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_411/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_411"
  op: "Mul"
  input: "Mul_411/x"
  input: "Mul_411/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_412/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_412"
  op: "Mul"
  input: "Mul_412/x"
  input: "CrossReplicaSum_72"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_257"
  op: "AddV2"
  input: "Mul_411"
  input: "Mul_412"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_413/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_413/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_413/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_413/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_413"
  op: "Mul"
  input: "Mul_413/x"
  input: "Mul_413/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_72"
  op: "Square"
  input: "CrossReplicaSum_72"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_414/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_414"
  op: "Mul"
  input: "Mul_414/x"
  input: "Square_72"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_258"
  op: "AddV2"
  input: "Mul_413"
  input: "Mul_414"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_72"
  op: "Sqrt"
  input: "add_258"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_259/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_259"
  op: "AddV2"
  input: "Sqrt_72"
  input: "add_259/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_75"
  op: "RealDiv"
  input: "add_257"
  input: "add_259"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_415"
  op: "Mul"
  input: "mul_21"
  input: "truediv_75"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_317"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_76"
  op: "Sub"
  input: "ReadVariableOp_317"
  input: "mul_415"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_216"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_76"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_318"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_216"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_217"
  op: "AssignVariableOp"
  input: "Mul_411/ReadVariableOp/Enter"
  input: "add_257"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_319"
  op: "ReadVariableOp"
  input: "Mul_411/ReadVariableOp/Enter"
  input: "^AssignVariableOp_217"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_218"
  op: "AssignVariableOp"
  input: "Mul_413/ReadVariableOp/Enter"
  input: "add_258"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_320"
  op: "ReadVariableOp"
  input: "Mul_413/ReadVariableOp/Enter"
  input: "^AssignVariableOp_218"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_416/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_416/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_416/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_416/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_416"
  op: "Mul"
  input: "Mul_416/x"
  input: "Mul_416/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_417/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_417"
  op: "Mul"
  input: "Mul_417/x"
  input: "CrossReplicaSum_73"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_260"
  op: "AddV2"
  input: "Mul_416"
  input: "Mul_417"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_418/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_418/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_418/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_418/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_418"
  op: "Mul"
  input: "Mul_418/x"
  input: "Mul_418/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_73"
  op: "Square"
  input: "CrossReplicaSum_73"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_419/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_419"
  op: "Mul"
  input: "Mul_419/x"
  input: "Square_73"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_261"
  op: "AddV2"
  input: "Mul_418"
  input: "Mul_419"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_73"
  op: "Sqrt"
  input: "add_261"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_262/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_262"
  op: "AddV2"
  input: "Sqrt_73"
  input: "add_262/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_76"
  op: "RealDiv"
  input: "add_260"
  input: "add_262"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_321"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_420/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_420"
  op: "Mul"
  input: "mul_420/x"
  input: "ReadVariableOp_321"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_263"
  op: "AddV2"
  input: "truediv_76"
  input: "mul_420"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_421"
  op: "Mul"
  input: "mul_21"
  input: "add_263"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_322"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_77"
  op: "Sub"
  input: "ReadVariableOp_322"
  input: "mul_421"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_219"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_77"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_323"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_219"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_220"
  op: "AssignVariableOp"
  input: "Mul_416/ReadVariableOp/Enter"
  input: "add_260"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_324"
  op: "ReadVariableOp"
  input: "Mul_416/ReadVariableOp/Enter"
  input: "^AssignVariableOp_220"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_221"
  op: "AssignVariableOp"
  input: "Mul_418/ReadVariableOp/Enter"
  input: "add_261"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_325"
  op: "ReadVariableOp"
  input: "Mul_418/ReadVariableOp/Enter"
  input: "^AssignVariableOp_221"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_422/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_422/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_422/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_422/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_422"
  op: "Mul"
  input: "Mul_422/x"
  input: "Mul_422/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_423/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_423"
  op: "Mul"
  input: "Mul_423/x"
  input: "CrossReplicaSum_74"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_264"
  op: "AddV2"
  input: "Mul_422"
  input: "Mul_423"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_424/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_424/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_424/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_424/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_424"
  op: "Mul"
  input: "Mul_424/x"
  input: "Mul_424/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_74"
  op: "Square"
  input: "CrossReplicaSum_74"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_425/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_425"
  op: "Mul"
  input: "Mul_425/x"
  input: "Square_74"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_265"
  op: "AddV2"
  input: "Mul_424"
  input: "Mul_425"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_74"
  op: "Sqrt"
  input: "add_265"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_266/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_266"
  op: "AddV2"
  input: "Sqrt_74"
  input: "add_266/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_77"
  op: "RealDiv"
  input: "add_264"
  input: "add_266"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_426"
  op: "Mul"
  input: "mul_21"
  input: "truediv_77"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_326"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_78"
  op: "Sub"
  input: "ReadVariableOp_326"
  input: "mul_426"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_222"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_78"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_327"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_222"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_223"
  op: "AssignVariableOp"
  input: "Mul_422/ReadVariableOp/Enter"
  input: "add_264"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_328"
  op: "ReadVariableOp"
  input: "Mul_422/ReadVariableOp/Enter"
  input: "^AssignVariableOp_223"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_224"
  op: "AssignVariableOp"
  input: "Mul_424/ReadVariableOp/Enter"
  input: "add_265"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_329"
  op: "ReadVariableOp"
  input: "Mul_424/ReadVariableOp/Enter"
  input: "^AssignVariableOp_224"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_427/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_427/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_427/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_427/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_427"
  op: "Mul"
  input: "Mul_427/x"
  input: "Mul_427/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_428/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_428"
  op: "Mul"
  input: "Mul_428/x"
  input: "CrossReplicaSum_75"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_267"
  op: "AddV2"
  input: "Mul_427"
  input: "Mul_428"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_429/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_429/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_429/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_429/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_429"
  op: "Mul"
  input: "Mul_429/x"
  input: "Mul_429/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_75"
  op: "Square"
  input: "CrossReplicaSum_75"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_430/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_430"
  op: "Mul"
  input: "Mul_430/x"
  input: "Square_75"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_268"
  op: "AddV2"
  input: "Mul_429"
  input: "Mul_430"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_75"
  op: "Sqrt"
  input: "add_268"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_269/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_269"
  op: "AddV2"
  input: "Sqrt_75"
  input: "add_269/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_78"
  op: "RealDiv"
  input: "add_267"
  input: "add_269"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_330"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_431/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_431"
  op: "Mul"
  input: "mul_431/x"
  input: "ReadVariableOp_330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_270"
  op: "AddV2"
  input: "truediv_78"
  input: "mul_431"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_432"
  op: "Mul"
  input: "mul_21"
  input: "add_270"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_331"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_79"
  op: "Sub"
  input: "ReadVariableOp_331"
  input: "mul_432"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_225"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_79"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_332"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_225"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_226"
  op: "AssignVariableOp"
  input: "Mul_427/ReadVariableOp/Enter"
  input: "add_267"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_333"
  op: "ReadVariableOp"
  input: "Mul_427/ReadVariableOp/Enter"
  input: "^AssignVariableOp_226"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_227"
  op: "AssignVariableOp"
  input: "Mul_429/ReadVariableOp/Enter"
  input: "add_268"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_334"
  op: "ReadVariableOp"
  input: "Mul_429/ReadVariableOp/Enter"
  input: "^AssignVariableOp_227"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_433/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_433/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_433/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_433/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_433"
  op: "Mul"
  input: "Mul_433/x"
  input: "Mul_433/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_434/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_434"
  op: "Mul"
  input: "Mul_434/x"
  input: "CrossReplicaSum_76"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_271"
  op: "AddV2"
  input: "Mul_433"
  input: "Mul_434"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_435/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_435/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_435/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_435/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_435"
  op: "Mul"
  input: "Mul_435/x"
  input: "Mul_435/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_76"
  op: "Square"
  input: "CrossReplicaSum_76"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_436/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_436"
  op: "Mul"
  input: "Mul_436/x"
  input: "Square_76"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_272"
  op: "AddV2"
  input: "Mul_435"
  input: "Mul_436"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_76"
  op: "Sqrt"
  input: "add_272"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_273/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_273"
  op: "AddV2"
  input: "Sqrt_76"
  input: "add_273/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_79"
  op: "RealDiv"
  input: "add_271"
  input: "add_273"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_437"
  op: "Mul"
  input: "mul_21"
  input: "truediv_79"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_335"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_80"
  op: "Sub"
  input: "ReadVariableOp_335"
  input: "mul_437"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_228"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_80"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_336"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_228"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_229"
  op: "AssignVariableOp"
  input: "Mul_433/ReadVariableOp/Enter"
  input: "add_271"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_337"
  op: "ReadVariableOp"
  input: "Mul_433/ReadVariableOp/Enter"
  input: "^AssignVariableOp_229"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_230"
  op: "AssignVariableOp"
  input: "Mul_435/ReadVariableOp/Enter"
  input: "add_272"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_338"
  op: "ReadVariableOp"
  input: "Mul_435/ReadVariableOp/Enter"
  input: "^AssignVariableOp_230"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_438/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_438/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_438/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_438/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_438"
  op: "Mul"
  input: "Mul_438/x"
  input: "Mul_438/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_439/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_439"
  op: "Mul"
  input: "Mul_439/x"
  input: "CrossReplicaSum_77"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_274"
  op: "AddV2"
  input: "Mul_438"
  input: "Mul_439"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_440/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_440/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_440/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_440/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_440"
  op: "Mul"
  input: "Mul_440/x"
  input: "Mul_440/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_77"
  op: "Square"
  input: "CrossReplicaSum_77"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_441/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_441"
  op: "Mul"
  input: "Mul_441/x"
  input: "Square_77"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_275"
  op: "AddV2"
  input: "Mul_440"
  input: "Mul_441"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_77"
  op: "Sqrt"
  input: "add_275"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_276/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_276"
  op: "AddV2"
  input: "Sqrt_77"
  input: "add_276/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_80"
  op: "RealDiv"
  input: "add_274"
  input: "add_276"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_339"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_442/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_442"
  op: "Mul"
  input: "mul_442/x"
  input: "ReadVariableOp_339"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_277"
  op: "AddV2"
  input: "truediv_80"
  input: "mul_442"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_443"
  op: "Mul"
  input: "mul_21"
  input: "add_277"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_340"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_81"
  op: "Sub"
  input: "ReadVariableOp_340"
  input: "mul_443"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_231"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_81"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_341"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_231"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_232"
  op: "AssignVariableOp"
  input: "Mul_438/ReadVariableOp/Enter"
  input: "add_274"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_342"
  op: "ReadVariableOp"
  input: "Mul_438/ReadVariableOp/Enter"
  input: "^AssignVariableOp_232"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_233"
  op: "AssignVariableOp"
  input: "Mul_440/ReadVariableOp/Enter"
  input: "add_275"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_343"
  op: "ReadVariableOp"
  input: "Mul_440/ReadVariableOp/Enter"
  input: "^AssignVariableOp_233"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_444/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_444/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_444/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_444/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_444"
  op: "Mul"
  input: "Mul_444/x"
  input: "Mul_444/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_445/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_445"
  op: "Mul"
  input: "Mul_445/x"
  input: "CrossReplicaSum_78"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_278"
  op: "AddV2"
  input: "Mul_444"
  input: "Mul_445"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_446/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_446/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_446/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_446/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_446"
  op: "Mul"
  input: "Mul_446/x"
  input: "Mul_446/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_78"
  op: "Square"
  input: "CrossReplicaSum_78"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_447/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_447"
  op: "Mul"
  input: "Mul_447/x"
  input: "Square_78"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_279"
  op: "AddV2"
  input: "Mul_446"
  input: "Mul_447"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_78"
  op: "Sqrt"
  input: "add_279"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_280/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_280"
  op: "AddV2"
  input: "Sqrt_78"
  input: "add_280/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_81"
  op: "RealDiv"
  input: "add_278"
  input: "add_280"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_448"
  op: "Mul"
  input: "mul_21"
  input: "truediv_81"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_344"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_82"
  op: "Sub"
  input: "ReadVariableOp_344"
  input: "mul_448"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_234"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_82"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_345"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_234"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_235"
  op: "AssignVariableOp"
  input: "Mul_444/ReadVariableOp/Enter"
  input: "add_278"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_346"
  op: "ReadVariableOp"
  input: "Mul_444/ReadVariableOp/Enter"
  input: "^AssignVariableOp_235"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_236"
  op: "AssignVariableOp"
  input: "Mul_446/ReadVariableOp/Enter"
  input: "add_279"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_347"
  op: "ReadVariableOp"
  input: "Mul_446/ReadVariableOp/Enter"
  input: "^AssignVariableOp_236"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_449/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_449/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_449/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_449/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_449"
  op: "Mul"
  input: "Mul_449/x"
  input: "Mul_449/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_450/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_450"
  op: "Mul"
  input: "Mul_450/x"
  input: "CrossReplicaSum_79"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_281"
  op: "AddV2"
  input: "Mul_449"
  input: "Mul_450"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_451/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_451/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_451/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_451/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_451"
  op: "Mul"
  input: "Mul_451/x"
  input: "Mul_451/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_79"
  op: "Square"
  input: "CrossReplicaSum_79"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_452/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_452"
  op: "Mul"
  input: "Mul_452/x"
  input: "Square_79"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_282"
  op: "AddV2"
  input: "Mul_451"
  input: "Mul_452"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_79"
  op: "Sqrt"
  input: "add_282"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_283/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_283"
  op: "AddV2"
  input: "Sqrt_79"
  input: "add_283/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_82"
  op: "RealDiv"
  input: "add_281"
  input: "add_283"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_453"
  op: "Mul"
  input: "mul_21"
  input: "truediv_82"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_348"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_83"
  op: "Sub"
  input: "ReadVariableOp_348"
  input: "mul_453"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_237"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_83"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_349"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_237"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_238"
  op: "AssignVariableOp"
  input: "Mul_449/ReadVariableOp/Enter"
  input: "add_281"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_350"
  op: "ReadVariableOp"
  input: "Mul_449/ReadVariableOp/Enter"
  input: "^AssignVariableOp_238"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_239"
  op: "AssignVariableOp"
  input: "Mul_451/ReadVariableOp/Enter"
  input: "add_282"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_351"
  op: "ReadVariableOp"
  input: "Mul_451/ReadVariableOp/Enter"
  input: "^AssignVariableOp_239"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_454/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_454/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_454/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_454/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_454"
  op: "Mul"
  input: "Mul_454/x"
  input: "Mul_454/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_455/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_455"
  op: "Mul"
  input: "Mul_455/x"
  input: "CrossReplicaSum_80"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_284"
  op: "AddV2"
  input: "Mul_454"
  input: "Mul_455"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_456/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_456/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_456/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_456/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_456"
  op: "Mul"
  input: "Mul_456/x"
  input: "Mul_456/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_80"
  op: "Square"
  input: "CrossReplicaSum_80"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_457/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_457"
  op: "Mul"
  input: "Mul_457/x"
  input: "Square_80"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_285"
  op: "AddV2"
  input: "Mul_456"
  input: "Mul_457"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_80"
  op: "Sqrt"
  input: "add_285"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_286/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_286"
  op: "AddV2"
  input: "Sqrt_80"
  input: "add_286/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_83"
  op: "RealDiv"
  input: "add_284"
  input: "add_286"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_458"
  op: "Mul"
  input: "mul_21"
  input: "truediv_83"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_352"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_84"
  op: "Sub"
  input: "ReadVariableOp_352"
  input: "mul_458"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_240"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_84"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_353"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_240"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_241"
  op: "AssignVariableOp"
  input: "Mul_454/ReadVariableOp/Enter"
  input: "add_284"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_354"
  op: "ReadVariableOp"
  input: "Mul_454/ReadVariableOp/Enter"
  input: "^AssignVariableOp_241"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_242"
  op: "AssignVariableOp"
  input: "Mul_456/ReadVariableOp/Enter"
  input: "add_285"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_355"
  op: "ReadVariableOp"
  input: "Mul_456/ReadVariableOp/Enter"
  input: "^AssignVariableOp_242"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_459/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_459/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_459/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_459/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_459"
  op: "Mul"
  input: "Mul_459/x"
  input: "Mul_459/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_460/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_460"
  op: "Mul"
  input: "Mul_460/x"
  input: "CrossReplicaSum_81"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_287"
  op: "AddV2"
  input: "Mul_459"
  input: "Mul_460"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_461/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_461/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_461/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_461/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_461"
  op: "Mul"
  input: "Mul_461/x"
  input: "Mul_461/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_81"
  op: "Square"
  input: "CrossReplicaSum_81"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_462/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_462"
  op: "Mul"
  input: "Mul_462/x"
  input: "Square_81"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_288"
  op: "AddV2"
  input: "Mul_461"
  input: "Mul_462"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_81"
  op: "Sqrt"
  input: "add_288"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_289/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_289"
  op: "AddV2"
  input: "Sqrt_81"
  input: "add_289/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_84"
  op: "RealDiv"
  input: "add_287"
  input: "add_289"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_356"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_463/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_463"
  op: "Mul"
  input: "mul_463/x"
  input: "ReadVariableOp_356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_290"
  op: "AddV2"
  input: "truediv_84"
  input: "mul_463"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_464"
  op: "Mul"
  input: "mul_21"
  input: "add_290"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_357"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_85"
  op: "Sub"
  input: "ReadVariableOp_357"
  input: "mul_464"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_243"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_85"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_358"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_243"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_244"
  op: "AssignVariableOp"
  input: "Mul_459/ReadVariableOp/Enter"
  input: "add_287"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_359"
  op: "ReadVariableOp"
  input: "Mul_459/ReadVariableOp/Enter"
  input: "^AssignVariableOp_244"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_245"
  op: "AssignVariableOp"
  input: "Mul_461/ReadVariableOp/Enter"
  input: "add_288"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_360"
  op: "ReadVariableOp"
  input: "Mul_461/ReadVariableOp/Enter"
  input: "^AssignVariableOp_245"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_465/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_465/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_465/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_465/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_465"
  op: "Mul"
  input: "Mul_465/x"
  input: "Mul_465/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_466/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_466"
  op: "Mul"
  input: "Mul_466/x"
  input: "CrossReplicaSum_82"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_291"
  op: "AddV2"
  input: "Mul_465"
  input: "Mul_466"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_467/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_467/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_467/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_467/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_467"
  op: "Mul"
  input: "Mul_467/x"
  input: "Mul_467/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_82"
  op: "Square"
  input: "CrossReplicaSum_82"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_468/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_468"
  op: "Mul"
  input: "Mul_468/x"
  input: "Square_82"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_292"
  op: "AddV2"
  input: "Mul_467"
  input: "Mul_468"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_82"
  op: "Sqrt"
  input: "add_292"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_293/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_293"
  op: "AddV2"
  input: "Sqrt_82"
  input: "add_293/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_85"
  op: "RealDiv"
  input: "add_291"
  input: "add_293"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_469"
  op: "Mul"
  input: "mul_21"
  input: "truediv_85"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_361"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_86"
  op: "Sub"
  input: "ReadVariableOp_361"
  input: "mul_469"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_246"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_86"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_362"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_246"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_247"
  op: "AssignVariableOp"
  input: "Mul_465/ReadVariableOp/Enter"
  input: "add_291"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_363"
  op: "ReadVariableOp"
  input: "Mul_465/ReadVariableOp/Enter"
  input: "^AssignVariableOp_247"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_248"
  op: "AssignVariableOp"
  input: "Mul_467/ReadVariableOp/Enter"
  input: "add_292"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_364"
  op: "ReadVariableOp"
  input: "Mul_467/ReadVariableOp/Enter"
  input: "^AssignVariableOp_248"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_470/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_470/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_470/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_470/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_470"
  op: "Mul"
  input: "Mul_470/x"
  input: "Mul_470/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_471/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_471"
  op: "Mul"
  input: "Mul_471/x"
  input: "CrossReplicaSum_83"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_294"
  op: "AddV2"
  input: "Mul_470"
  input: "Mul_471"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_472/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_472/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_472/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_472/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_472"
  op: "Mul"
  input: "Mul_472/x"
  input: "Mul_472/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_83"
  op: "Square"
  input: "CrossReplicaSum_83"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_473/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_473"
  op: "Mul"
  input: "Mul_473/x"
  input: "Square_83"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_295"
  op: "AddV2"
  input: "Mul_472"
  input: "Mul_473"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_83"
  op: "Sqrt"
  input: "add_295"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_296/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_296"
  op: "AddV2"
  input: "Sqrt_83"
  input: "add_296/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_86"
  op: "RealDiv"
  input: "add_294"
  input: "add_296"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_365"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_474/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_474"
  op: "Mul"
  input: "mul_474/x"
  input: "ReadVariableOp_365"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_297"
  op: "AddV2"
  input: "truediv_86"
  input: "mul_474"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_475"
  op: "Mul"
  input: "mul_21"
  input: "add_297"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_366"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_87"
  op: "Sub"
  input: "ReadVariableOp_366"
  input: "mul_475"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_249"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_87"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_367"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_249"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_250"
  op: "AssignVariableOp"
  input: "Mul_470/ReadVariableOp/Enter"
  input: "add_294"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_368"
  op: "ReadVariableOp"
  input: "Mul_470/ReadVariableOp/Enter"
  input: "^AssignVariableOp_250"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_251"
  op: "AssignVariableOp"
  input: "Mul_472/ReadVariableOp/Enter"
  input: "add_295"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_369"
  op: "ReadVariableOp"
  input: "Mul_472/ReadVariableOp/Enter"
  input: "^AssignVariableOp_251"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_m"
  input: "generator/encoder/layer_4/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_v"
  input: "generator/encoder/layer_4/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_476/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_476/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_476/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_476/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_476"
  op: "Mul"
  input: "Mul_476/x"
  input: "Mul_476/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_477/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_477"
  op: "Mul"
  input: "Mul_477/x"
  input: "CrossReplicaSum_84"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_298"
  op: "AddV2"
  input: "Mul_476"
  input: "Mul_477"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_478/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_478/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_478/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_478/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_478"
  op: "Mul"
  input: "Mul_478/x"
  input: "Mul_478/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_84"
  op: "Square"
  input: "CrossReplicaSum_84"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_479/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_479"
  op: "Mul"
  input: "Mul_479/x"
  input: "Square_84"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_299"
  op: "AddV2"
  input: "Mul_478"
  input: "Mul_479"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_84"
  op: "Sqrt"
  input: "add_299"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_300/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_300"
  op: "AddV2"
  input: "Sqrt_84"
  input: "add_300/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_87"
  op: "RealDiv"
  input: "add_298"
  input: "add_300"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_480"
  op: "Mul"
  input: "mul_21"
  input: "truediv_87"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_370"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_88"
  op: "Sub"
  input: "ReadVariableOp_370"
  input: "mul_480"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_252"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_88"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_371"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_252"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_253"
  op: "AssignVariableOp"
  input: "Mul_476/ReadVariableOp/Enter"
  input: "add_298"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_372"
  op: "ReadVariableOp"
  input: "Mul_476/ReadVariableOp/Enter"
  input: "^AssignVariableOp_253"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_254"
  op: "AssignVariableOp"
  input: "Mul_478/ReadVariableOp/Enter"
  input: "add_299"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_373"
  op: "ReadVariableOp"
  input: "Mul_478/ReadVariableOp/Enter"
  input: "^AssignVariableOp_254"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_481/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_481/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_481/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_481/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_481"
  op: "Mul"
  input: "Mul_481/x"
  input: "Mul_481/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_482/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_482"
  op: "Mul"
  input: "Mul_482/x"
  input: "CrossReplicaSum_85"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_301"
  op: "AddV2"
  input: "Mul_481"
  input: "Mul_482"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_483/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_483/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_483/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_483/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_483"
  op: "Mul"
  input: "Mul_483/x"
  input: "Mul_483/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_85"
  op: "Square"
  input: "CrossReplicaSum_85"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_484/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_484"
  op: "Mul"
  input: "Mul_484/x"
  input: "Square_85"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_302"
  op: "AddV2"
  input: "Mul_483"
  input: "Mul_484"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_85"
  op: "Sqrt"
  input: "add_302"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_303/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_303"
  op: "AddV2"
  input: "Sqrt_85"
  input: "add_303/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_88"
  op: "RealDiv"
  input: "add_301"
  input: "add_303"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_485"
  op: "Mul"
  input: "mul_21"
  input: "truediv_88"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_374"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_89"
  op: "Sub"
  input: "ReadVariableOp_374"
  input: "mul_485"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_255"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_89"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_375"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_255"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_256"
  op: "AssignVariableOp"
  input: "Mul_481/ReadVariableOp/Enter"
  input: "add_301"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_376"
  op: "ReadVariableOp"
  input: "Mul_481/ReadVariableOp/Enter"
  input: "^AssignVariableOp_256"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_257"
  op: "AssignVariableOp"
  input: "Mul_483/ReadVariableOp/Enter"
  input: "add_302"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_377"
  op: "ReadVariableOp"
  input: "Mul_483/ReadVariableOp/Enter"
  input: "^AssignVariableOp_257"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_486/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_486/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_486/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_486/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_486"
  op: "Mul"
  input: "Mul_486/x"
  input: "Mul_486/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_487/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_487"
  op: "Mul"
  input: "Mul_487/x"
  input: "CrossReplicaSum_86"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_304"
  op: "AddV2"
  input: "Mul_486"
  input: "Mul_487"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_488/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_488/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_488/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_488/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_488"
  op: "Mul"
  input: "Mul_488/x"
  input: "Mul_488/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_86"
  op: "Square"
  input: "CrossReplicaSum_86"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_489/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_489"
  op: "Mul"
  input: "Mul_489/x"
  input: "Square_86"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_305"
  op: "AddV2"
  input: "Mul_488"
  input: "Mul_489"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_86"
  op: "Sqrt"
  input: "add_305"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_306/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_306"
  op: "AddV2"
  input: "Sqrt_86"
  input: "add_306/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_89"
  op: "RealDiv"
  input: "add_304"
  input: "add_306"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_490"
  op: "Mul"
  input: "mul_21"
  input: "truediv_89"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_378"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_90"
  op: "Sub"
  input: "ReadVariableOp_378"
  input: "mul_490"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_258"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_90"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_379"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_258"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_259"
  op: "AssignVariableOp"
  input: "Mul_486/ReadVariableOp/Enter"
  input: "add_304"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_380"
  op: "ReadVariableOp"
  input: "Mul_486/ReadVariableOp/Enter"
  input: "^AssignVariableOp_259"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_260"
  op: "AssignVariableOp"
  input: "Mul_488/ReadVariableOp/Enter"
  input: "add_305"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_381"
  op: "ReadVariableOp"
  input: "Mul_488/ReadVariableOp/Enter"
  input: "^AssignVariableOp_260"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_491/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_491/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_491/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_491/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_491"
  op: "Mul"
  input: "Mul_491/x"
  input: "Mul_491/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_492/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_492"
  op: "Mul"
  input: "Mul_492/x"
  input: "CrossReplicaSum_87"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_307"
  op: "AddV2"
  input: "Mul_491"
  input: "Mul_492"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_493/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_493/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_493/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_493/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_493"
  op: "Mul"
  input: "Mul_493/x"
  input: "Mul_493/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_87"
  op: "Square"
  input: "CrossReplicaSum_87"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_494/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_494"
  op: "Mul"
  input: "Mul_494/x"
  input: "Square_87"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_308"
  op: "AddV2"
  input: "Mul_493"
  input: "Mul_494"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_87"
  op: "Sqrt"
  input: "add_308"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_309/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_309"
  op: "AddV2"
  input: "Sqrt_87"
  input: "add_309/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_90"
  op: "RealDiv"
  input: "add_307"
  input: "add_309"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_382"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_495/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_495"
  op: "Mul"
  input: "mul_495/x"
  input: "ReadVariableOp_382"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_310"
  op: "AddV2"
  input: "truediv_90"
  input: "mul_495"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_496"
  op: "Mul"
  input: "mul_21"
  input: "add_310"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_383"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_91"
  op: "Sub"
  input: "ReadVariableOp_383"
  input: "mul_496"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_261"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_91"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_384"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_261"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_262"
  op: "AssignVariableOp"
  input: "Mul_491/ReadVariableOp/Enter"
  input: "add_307"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_385"
  op: "ReadVariableOp"
  input: "Mul_491/ReadVariableOp/Enter"
  input: "^AssignVariableOp_262"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_263"
  op: "AssignVariableOp"
  input: "Mul_493/ReadVariableOp/Enter"
  input: "add_308"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_386"
  op: "ReadVariableOp"
  input: "Mul_493/ReadVariableOp/Enter"
  input: "^AssignVariableOp_263"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_497/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_497/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_497/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_497/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_497"
  op: "Mul"
  input: "Mul_497/x"
  input: "Mul_497/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_498/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_498"
  op: "Mul"
  input: "Mul_498/x"
  input: "CrossReplicaSum_88"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_311"
  op: "AddV2"
  input: "Mul_497"
  input: "Mul_498"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_499/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_499/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_499/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_499/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_499"
  op: "Mul"
  input: "Mul_499/x"
  input: "Mul_499/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_88"
  op: "Square"
  input: "CrossReplicaSum_88"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_500/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_500"
  op: "Mul"
  input: "Mul_500/x"
  input: "Square_88"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_312"
  op: "AddV2"
  input: "Mul_499"
  input: "Mul_500"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_88"
  op: "Sqrt"
  input: "add_312"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_313/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_313"
  op: "AddV2"
  input: "Sqrt_88"
  input: "add_313/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_91"
  op: "RealDiv"
  input: "add_311"
  input: "add_313"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_501"
  op: "Mul"
  input: "mul_21"
  input: "truediv_91"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_387"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_92"
  op: "Sub"
  input: "ReadVariableOp_387"
  input: "mul_501"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_264"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_92"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_388"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_264"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_265"
  op: "AssignVariableOp"
  input: "Mul_497/ReadVariableOp/Enter"
  input: "add_311"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_389"
  op: "ReadVariableOp"
  input: "Mul_497/ReadVariableOp/Enter"
  input: "^AssignVariableOp_265"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_266"
  op: "AssignVariableOp"
  input: "Mul_499/ReadVariableOp/Enter"
  input: "add_312"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_390"
  op: "ReadVariableOp"
  input: "Mul_499/ReadVariableOp/Enter"
  input: "^AssignVariableOp_266"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_502/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_502/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_502/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_502/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_502"
  op: "Mul"
  input: "Mul_502/x"
  input: "Mul_502/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_503/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_503"
  op: "Mul"
  input: "Mul_503/x"
  input: "CrossReplicaSum_89"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_314"
  op: "AddV2"
  input: "Mul_502"
  input: "Mul_503"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_504/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_504/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_504/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_504/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_504"
  op: "Mul"
  input: "Mul_504/x"
  input: "Mul_504/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_89"
  op: "Square"
  input: "CrossReplicaSum_89"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_505/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_505"
  op: "Mul"
  input: "Mul_505/x"
  input: "Square_89"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_315"
  op: "AddV2"
  input: "Mul_504"
  input: "Mul_505"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_89"
  op: "Sqrt"
  input: "add_315"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_316/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_316"
  op: "AddV2"
  input: "Sqrt_89"
  input: "add_316/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_92"
  op: "RealDiv"
  input: "add_314"
  input: "add_316"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_391"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_506/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_506"
  op: "Mul"
  input: "mul_506/x"
  input: "ReadVariableOp_391"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_317"
  op: "AddV2"
  input: "truediv_92"
  input: "mul_506"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_507"
  op: "Mul"
  input: "mul_21"
  input: "add_317"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_392"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_93"
  op: "Sub"
  input: "ReadVariableOp_392"
  input: "mul_507"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_267"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_93"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_393"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_267"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_268"
  op: "AssignVariableOp"
  input: "Mul_502/ReadVariableOp/Enter"
  input: "add_314"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_394"
  op: "ReadVariableOp"
  input: "Mul_502/ReadVariableOp/Enter"
  input: "^AssignVariableOp_268"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_269"
  op: "AssignVariableOp"
  input: "Mul_504/ReadVariableOp/Enter"
  input: "add_315"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_395"
  op: "ReadVariableOp"
  input: "Mul_504/ReadVariableOp/Enter"
  input: "^AssignVariableOp_269"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_508/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_508/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_508/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_508/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_508"
  op: "Mul"
  input: "Mul_508/x"
  input: "Mul_508/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_509/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_509"
  op: "Mul"
  input: "Mul_509/x"
  input: "CrossReplicaSum_90"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_318"
  op: "AddV2"
  input: "Mul_508"
  input: "Mul_509"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_510/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_510/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_510/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_510/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_510"
  op: "Mul"
  input: "Mul_510/x"
  input: "Mul_510/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_90"
  op: "Square"
  input: "CrossReplicaSum_90"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_511/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_511"
  op: "Mul"
  input: "Mul_511/x"
  input: "Square_90"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_319"
  op: "AddV2"
  input: "Mul_510"
  input: "Mul_511"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_90"
  op: "Sqrt"
  input: "add_319"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_320/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_320"
  op: "AddV2"
  input: "Sqrt_90"
  input: "add_320/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_93"
  op: "RealDiv"
  input: "add_318"
  input: "add_320"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_512"
  op: "Mul"
  input: "mul_21"
  input: "truediv_93"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_396"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_94"
  op: "Sub"
  input: "ReadVariableOp_396"
  input: "mul_512"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_270"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_94"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_397"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_270"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_271"
  op: "AssignVariableOp"
  input: "Mul_508/ReadVariableOp/Enter"
  input: "add_318"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_398"
  op: "ReadVariableOp"
  input: "Mul_508/ReadVariableOp/Enter"
  input: "^AssignVariableOp_271"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_272"
  op: "AssignVariableOp"
  input: "Mul_510/ReadVariableOp/Enter"
  input: "add_319"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_399"
  op: "ReadVariableOp"
  input: "Mul_510/ReadVariableOp/Enter"
  input: "^AssignVariableOp_272"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_513/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_513/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_513/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_513/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_513"
  op: "Mul"
  input: "Mul_513/x"
  input: "Mul_513/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_514/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_514"
  op: "Mul"
  input: "Mul_514/x"
  input: "CrossReplicaSum_91"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_321"
  op: "AddV2"
  input: "Mul_513"
  input: "Mul_514"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_515/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_515/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_515/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_515/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_515"
  op: "Mul"
  input: "Mul_515/x"
  input: "Mul_515/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_91"
  op: "Square"
  input: "CrossReplicaSum_91"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_516/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_516"
  op: "Mul"
  input: "Mul_516/x"
  input: "Square_91"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_322"
  op: "AddV2"
  input: "Mul_515"
  input: "Mul_516"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_91"
  op: "Sqrt"
  input: "add_322"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_323/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_323"
  op: "AddV2"
  input: "Sqrt_91"
  input: "add_323/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_94"
  op: "RealDiv"
  input: "add_321"
  input: "add_323"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_400"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_517/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_517"
  op: "Mul"
  input: "mul_517/x"
  input: "ReadVariableOp_400"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_324"
  op: "AddV2"
  input: "truediv_94"
  input: "mul_517"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_518"
  op: "Mul"
  input: "mul_21"
  input: "add_324"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_401"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_95"
  op: "Sub"
  input: "ReadVariableOp_401"
  input: "mul_518"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_273"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_95"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_402"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_273"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_274"
  op: "AssignVariableOp"
  input: "Mul_513/ReadVariableOp/Enter"
  input: "add_321"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_403"
  op: "ReadVariableOp"
  input: "Mul_513/ReadVariableOp/Enter"
  input: "^AssignVariableOp_274"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_275"
  op: "AssignVariableOp"
  input: "Mul_515/ReadVariableOp/Enter"
  input: "add_322"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_404"
  op: "ReadVariableOp"
  input: "Mul_515/ReadVariableOp/Enter"
  input: "^AssignVariableOp_275"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_519/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_519/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_519/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_519/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_519"
  op: "Mul"
  input: "Mul_519/x"
  input: "Mul_519/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_520/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_520"
  op: "Mul"
  input: "Mul_520/x"
  input: "CrossReplicaSum_92"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_325"
  op: "AddV2"
  input: "Mul_519"
  input: "Mul_520"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_521/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_521/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_521/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_521/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_521"
  op: "Mul"
  input: "Mul_521/x"
  input: "Mul_521/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_92"
  op: "Square"
  input: "CrossReplicaSum_92"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_522/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_522"
  op: "Mul"
  input: "Mul_522/x"
  input: "Square_92"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_326"
  op: "AddV2"
  input: "Mul_521"
  input: "Mul_522"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_92"
  op: "Sqrt"
  input: "add_326"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_327/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_327"
  op: "AddV2"
  input: "Sqrt_92"
  input: "add_327/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_95"
  op: "RealDiv"
  input: "add_325"
  input: "add_327"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_523"
  op: "Mul"
  input: "mul_21"
  input: "truediv_95"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_405"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_96"
  op: "Sub"
  input: "ReadVariableOp_405"
  input: "mul_523"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_276"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_96"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_406"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_276"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_277"
  op: "AssignVariableOp"
  input: "Mul_519/ReadVariableOp/Enter"
  input: "add_325"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_407"
  op: "ReadVariableOp"
  input: "Mul_519/ReadVariableOp/Enter"
  input: "^AssignVariableOp_277"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_278"
  op: "AssignVariableOp"
  input: "Mul_521/ReadVariableOp/Enter"
  input: "add_326"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_408"
  op: "ReadVariableOp"
  input: "Mul_521/ReadVariableOp/Enter"
  input: "^AssignVariableOp_278"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_524/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_524/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_524/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_524/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_524"
  op: "Mul"
  input: "Mul_524/x"
  input: "Mul_524/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_525/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_525"
  op: "Mul"
  input: "Mul_525/x"
  input: "CrossReplicaSum_93"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_328"
  op: "AddV2"
  input: "Mul_524"
  input: "Mul_525"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_526/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_526/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_526/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_526/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_526"
  op: "Mul"
  input: "Mul_526/x"
  input: "Mul_526/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_93"
  op: "Square"
  input: "CrossReplicaSum_93"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_527/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_527"
  op: "Mul"
  input: "Mul_527/x"
  input: "Square_93"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_329"
  op: "AddV2"
  input: "Mul_526"
  input: "Mul_527"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_93"
  op: "Sqrt"
  input: "add_329"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_330/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_330"
  op: "AddV2"
  input: "Sqrt_93"
  input: "add_330/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_96"
  op: "RealDiv"
  input: "add_328"
  input: "add_330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_409"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_528/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_528"
  op: "Mul"
  input: "mul_528/x"
  input: "ReadVariableOp_409"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_331"
  op: "AddV2"
  input: "truediv_96"
  input: "mul_528"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_529"
  op: "Mul"
  input: "mul_21"
  input: "add_331"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_410"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_97"
  op: "Sub"
  input: "ReadVariableOp_410"
  input: "mul_529"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_279"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_97"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_411"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_279"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_280"
  op: "AssignVariableOp"
  input: "Mul_524/ReadVariableOp/Enter"
  input: "add_328"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_412"
  op: "ReadVariableOp"
  input: "Mul_524/ReadVariableOp/Enter"
  input: "^AssignVariableOp_280"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_281"
  op: "AssignVariableOp"
  input: "Mul_526/ReadVariableOp/Enter"
  input: "add_329"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_413"
  op: "ReadVariableOp"
  input: "Mul_526/ReadVariableOp/Enter"
  input: "^AssignVariableOp_281"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_530/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_530/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_530/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_530/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_530"
  op: "Mul"
  input: "Mul_530/x"
  input: "Mul_530/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_531/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_531"
  op: "Mul"
  input: "Mul_531/x"
  input: "CrossReplicaSum_94"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_332"
  op: "AddV2"
  input: "Mul_530"
  input: "Mul_531"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_532/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_532/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_532/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_532/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_532"
  op: "Mul"
  input: "Mul_532/x"
  input: "Mul_532/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_94"
  op: "Square"
  input: "CrossReplicaSum_94"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_533/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_533"
  op: "Mul"
  input: "Mul_533/x"
  input: "Square_94"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_333"
  op: "AddV2"
  input: "Mul_532"
  input: "Mul_533"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_94"
  op: "Sqrt"
  input: "add_333"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_334/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_334"
  op: "AddV2"
  input: "Sqrt_94"
  input: "add_334/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_97"
  op: "RealDiv"
  input: "add_332"
  input: "add_334"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_534"
  op: "Mul"
  input: "mul_21"
  input: "truediv_97"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_414"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_98"
  op: "Sub"
  input: "ReadVariableOp_414"
  input: "mul_534"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_282"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_98"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_415"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_282"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_283"
  op: "AssignVariableOp"
  input: "Mul_530/ReadVariableOp/Enter"
  input: "add_332"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_416"
  op: "ReadVariableOp"
  input: "Mul_530/ReadVariableOp/Enter"
  input: "^AssignVariableOp_283"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_284"
  op: "AssignVariableOp"
  input: "Mul_532/ReadVariableOp/Enter"
  input: "add_333"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_417"
  op: "ReadVariableOp"
  input: "Mul_532/ReadVariableOp/Enter"
  input: "^AssignVariableOp_284"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_535/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_535/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_535/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_535/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_535"
  op: "Mul"
  input: "Mul_535/x"
  input: "Mul_535/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_536/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_536"
  op: "Mul"
  input: "Mul_536/x"
  input: "CrossReplicaSum_95"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_335"
  op: "AddV2"
  input: "Mul_535"
  input: "Mul_536"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_537/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_537/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_537/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_537/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_537"
  op: "Mul"
  input: "Mul_537/x"
  input: "Mul_537/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_95"
  op: "Square"
  input: "CrossReplicaSum_95"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_538/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_538"
  op: "Mul"
  input: "Mul_538/x"
  input: "Square_95"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_336"
  op: "AddV2"
  input: "Mul_537"
  input: "Mul_538"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_95"
  op: "Sqrt"
  input: "add_336"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_337/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_337"
  op: "AddV2"
  input: "Sqrt_95"
  input: "add_337/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_98"
  op: "RealDiv"
  input: "add_335"
  input: "add_337"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_539"
  op: "Mul"
  input: "mul_21"
  input: "truediv_98"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_418"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_99"
  op: "Sub"
  input: "ReadVariableOp_418"
  input: "mul_539"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_285"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_99"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_419"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_285"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_286"
  op: "AssignVariableOp"
  input: "Mul_535/ReadVariableOp/Enter"
  input: "add_335"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_420"
  op: "ReadVariableOp"
  input: "Mul_535/ReadVariableOp/Enter"
  input: "^AssignVariableOp_286"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_287"
  op: "AssignVariableOp"
  input: "Mul_537/ReadVariableOp/Enter"
  input: "add_336"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_421"
  op: "ReadVariableOp"
  input: "Mul_537/ReadVariableOp/Enter"
  input: "^AssignVariableOp_287"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_540/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_540/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_540/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_540/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_540"
  op: "Mul"
  input: "Mul_540/x"
  input: "Mul_540/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_541/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_541"
  op: "Mul"
  input: "Mul_541/x"
  input: "CrossReplicaSum_96"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_338"
  op: "AddV2"
  input: "Mul_540"
  input: "Mul_541"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_542/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_542/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_542/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_542/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_542"
  op: "Mul"
  input: "Mul_542/x"
  input: "Mul_542/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_96"
  op: "Square"
  input: "CrossReplicaSum_96"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_543/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_543"
  op: "Mul"
  input: "Mul_543/x"
  input: "Square_96"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_339"
  op: "AddV2"
  input: "Mul_542"
  input: "Mul_543"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_96"
  op: "Sqrt"
  input: "add_339"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_340/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_340"
  op: "AddV2"
  input: "Sqrt_96"
  input: "add_340/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_99"
  op: "RealDiv"
  input: "add_338"
  input: "add_340"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_544"
  op: "Mul"
  input: "mul_21"
  input: "truediv_99"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_422"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_100"
  op: "Sub"
  input: "ReadVariableOp_422"
  input: "mul_544"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_288"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_100"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_423"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_288"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_289"
  op: "AssignVariableOp"
  input: "Mul_540/ReadVariableOp/Enter"
  input: "add_338"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_424"
  op: "ReadVariableOp"
  input: "Mul_540/ReadVariableOp/Enter"
  input: "^AssignVariableOp_289"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_290"
  op: "AssignVariableOp"
  input: "Mul_542/ReadVariableOp/Enter"
  input: "add_339"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_425"
  op: "ReadVariableOp"
  input: "Mul_542/ReadVariableOp/Enter"
  input: "^AssignVariableOp_290"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_545/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_545/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_545/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_545/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_545"
  op: "Mul"
  input: "Mul_545/x"
  input: "Mul_545/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_546/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_546"
  op: "Mul"
  input: "Mul_546/x"
  input: "CrossReplicaSum_97"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_341"
  op: "AddV2"
  input: "Mul_545"
  input: "Mul_546"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_547/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_547/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_547/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_547/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_547"
  op: "Mul"
  input: "Mul_547/x"
  input: "Mul_547/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_97"
  op: "Square"
  input: "CrossReplicaSum_97"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_548/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_548"
  op: "Mul"
  input: "Mul_548/x"
  input: "Square_97"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_342"
  op: "AddV2"
  input: "Mul_547"
  input: "Mul_548"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_97"
  op: "Sqrt"
  input: "add_342"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_343/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_343"
  op: "AddV2"
  input: "Sqrt_97"
  input: "add_343/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_100"
  op: "RealDiv"
  input: "add_341"
  input: "add_343"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_426"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_549/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_549"
  op: "Mul"
  input: "mul_549/x"
  input: "ReadVariableOp_426"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_344"
  op: "AddV2"
  input: "truediv_100"
  input: "mul_549"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_550"
  op: "Mul"
  input: "mul_21"
  input: "add_344"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_427"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_101"
  op: "Sub"
  input: "ReadVariableOp_427"
  input: "mul_550"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_291"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_101"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_428"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_291"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_292"
  op: "AssignVariableOp"
  input: "Mul_545/ReadVariableOp/Enter"
  input: "add_341"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_429"
  op: "ReadVariableOp"
  input: "Mul_545/ReadVariableOp/Enter"
  input: "^AssignVariableOp_292"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_293"
  op: "AssignVariableOp"
  input: "Mul_547/ReadVariableOp/Enter"
  input: "add_342"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_430"
  op: "ReadVariableOp"
  input: "Mul_547/ReadVariableOp/Enter"
  input: "^AssignVariableOp_293"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_551/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_551/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_551/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_551/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_551"
  op: "Mul"
  input: "Mul_551/x"
  input: "Mul_551/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_552/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_552"
  op: "Mul"
  input: "Mul_552/x"
  input: "CrossReplicaSum_98"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_345"
  op: "AddV2"
  input: "Mul_551"
  input: "Mul_552"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_553/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_553/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_553/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_553/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_553"
  op: "Mul"
  input: "Mul_553/x"
  input: "Mul_553/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_98"
  op: "Square"
  input: "CrossReplicaSum_98"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_554/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_554"
  op: "Mul"
  input: "Mul_554/x"
  input: "Square_98"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_346"
  op: "AddV2"
  input: "Mul_553"
  input: "Mul_554"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_98"
  op: "Sqrt"
  input: "add_346"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_347/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_347"
  op: "AddV2"
  input: "Sqrt_98"
  input: "add_347/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_101"
  op: "RealDiv"
  input: "add_345"
  input: "add_347"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_555"
  op: "Mul"
  input: "mul_21"
  input: "truediv_101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_431"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_102"
  op: "Sub"
  input: "ReadVariableOp_431"
  input: "mul_555"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_294"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_102"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_432"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_294"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_295"
  op: "AssignVariableOp"
  input: "Mul_551/ReadVariableOp/Enter"
  input: "add_345"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_433"
  op: "ReadVariableOp"
  input: "Mul_551/ReadVariableOp/Enter"
  input: "^AssignVariableOp_295"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_296"
  op: "AssignVariableOp"
  input: "Mul_553/ReadVariableOp/Enter"
  input: "add_346"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_434"
  op: "ReadVariableOp"
  input: "Mul_553/ReadVariableOp/Enter"
  input: "^AssignVariableOp_296"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_556/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_556/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_556/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_556/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_556"
  op: "Mul"
  input: "Mul_556/x"
  input: "Mul_556/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_557/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_557"
  op: "Mul"
  input: "Mul_557/x"
  input: "CrossReplicaSum_99"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_348"
  op: "AddV2"
  input: "Mul_556"
  input: "Mul_557"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_558/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_558/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_558/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_558/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_558"
  op: "Mul"
  input: "Mul_558/x"
  input: "Mul_558/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_99"
  op: "Square"
  input: "CrossReplicaSum_99"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_559/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_559"
  op: "Mul"
  input: "Mul_559/x"
  input: "Square_99"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_349"
  op: "AddV2"
  input: "Mul_558"
  input: "Mul_559"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_99"
  op: "Sqrt"
  input: "add_349"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_350/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_350"
  op: "AddV2"
  input: "Sqrt_99"
  input: "add_350/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_102"
  op: "RealDiv"
  input: "add_348"
  input: "add_350"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_435"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_560/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_560"
  op: "Mul"
  input: "mul_560/x"
  input: "ReadVariableOp_435"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_351"
  op: "AddV2"
  input: "truediv_102"
  input: "mul_560"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_561"
  op: "Mul"
  input: "mul_21"
  input: "add_351"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_436"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_103"
  op: "Sub"
  input: "ReadVariableOp_436"
  input: "mul_561"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_297"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_103"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_437"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_297"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_298"
  op: "AssignVariableOp"
  input: "Mul_556/ReadVariableOp/Enter"
  input: "add_348"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_438"
  op: "ReadVariableOp"
  input: "Mul_556/ReadVariableOp/Enter"
  input: "^AssignVariableOp_298"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_299"
  op: "AssignVariableOp"
  input: "Mul_558/ReadVariableOp/Enter"
  input: "add_349"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_439"
  op: "ReadVariableOp"
  input: "Mul_558/ReadVariableOp/Enter"
  input: "^AssignVariableOp_299"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_m"
  input: "generator/encoder/layer_5/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_v"
  input: "generator/encoder/layer_5/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_562/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_562/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_562/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_562/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_562"
  op: "Mul"
  input: "Mul_562/x"
  input: "Mul_562/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_563/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_563"
  op: "Mul"
  input: "Mul_563/x"
  input: "CrossReplicaSum_100"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_352"
  op: "AddV2"
  input: "Mul_562"
  input: "Mul_563"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_564/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_564/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_564/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_564/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_564"
  op: "Mul"
  input: "Mul_564/x"
  input: "Mul_564/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_100"
  op: "Square"
  input: "CrossReplicaSum_100"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_565/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_565"
  op: "Mul"
  input: "Mul_565/x"
  input: "Square_100"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_353"
  op: "AddV2"
  input: "Mul_564"
  input: "Mul_565"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_100"
  op: "Sqrt"
  input: "add_353"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_354/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_354"
  op: "AddV2"
  input: "Sqrt_100"
  input: "add_354/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_103"
  op: "RealDiv"
  input: "add_352"
  input: "add_354"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_566"
  op: "Mul"
  input: "mul_21"
  input: "truediv_103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_440"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_104"
  op: "Sub"
  input: "ReadVariableOp_440"
  input: "mul_566"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_300"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_104"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_441"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_300"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_301"
  op: "AssignVariableOp"
  input: "Mul_562/ReadVariableOp/Enter"
  input: "add_352"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_442"
  op: "ReadVariableOp"
  input: "Mul_562/ReadVariableOp/Enter"
  input: "^AssignVariableOp_301"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_302"
  op: "AssignVariableOp"
  input: "Mul_564/ReadVariableOp/Enter"
  input: "add_353"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_443"
  op: "ReadVariableOp"
  input: "Mul_564/ReadVariableOp/Enter"
  input: "^AssignVariableOp_302"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_567/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_567/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_567/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_567/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_567"
  op: "Mul"
  input: "Mul_567/x"
  input: "Mul_567/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_568/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_568"
  op: "Mul"
  input: "Mul_568/x"
  input: "CrossReplicaSum_101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_355"
  op: "AddV2"
  input: "Mul_567"
  input: "Mul_568"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_569/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_569/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_569/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_569/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_569"
  op: "Mul"
  input: "Mul_569/x"
  input: "Mul_569/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_101"
  op: "Square"
  input: "CrossReplicaSum_101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_570/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_570"
  op: "Mul"
  input: "Mul_570/x"
  input: "Square_101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_356"
  op: "AddV2"
  input: "Mul_569"
  input: "Mul_570"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_101"
  op: "Sqrt"
  input: "add_356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_357/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_357"
  op: "AddV2"
  input: "Sqrt_101"
  input: "add_357/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_104"
  op: "RealDiv"
  input: "add_355"
  input: "add_357"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_571"
  op: "Mul"
  input: "mul_21"
  input: "truediv_104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_444"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_105"
  op: "Sub"
  input: "ReadVariableOp_444"
  input: "mul_571"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_303"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_105"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_445"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_303"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_304"
  op: "AssignVariableOp"
  input: "Mul_567/ReadVariableOp/Enter"
  input: "add_355"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_446"
  op: "ReadVariableOp"
  input: "Mul_567/ReadVariableOp/Enter"
  input: "^AssignVariableOp_304"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_305"
  op: "AssignVariableOp"
  input: "Mul_569/ReadVariableOp/Enter"
  input: "add_356"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_447"
  op: "ReadVariableOp"
  input: "Mul_569/ReadVariableOp/Enter"
  input: "^AssignVariableOp_305"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_572/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_572/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_572/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_572/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_572"
  op: "Mul"
  input: "Mul_572/x"
  input: "Mul_572/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_573/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_573"
  op: "Mul"
  input: "Mul_573/x"
  input: "CrossReplicaSum_102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_358"
  op: "AddV2"
  input: "Mul_572"
  input: "Mul_573"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_574/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_574/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_574/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_574/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_574"
  op: "Mul"
  input: "Mul_574/x"
  input: "Mul_574/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_102"
  op: "Square"
  input: "CrossReplicaSum_102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_575/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_575"
  op: "Mul"
  input: "Mul_575/x"
  input: "Square_102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_359"
  op: "AddV2"
  input: "Mul_574"
  input: "Mul_575"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_102"
  op: "Sqrt"
  input: "add_359"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_360/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_360"
  op: "AddV2"
  input: "Sqrt_102"
  input: "add_360/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_105"
  op: "RealDiv"
  input: "add_358"
  input: "add_360"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_576"
  op: "Mul"
  input: "mul_21"
  input: "truediv_105"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_448"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_106"
  op: "Sub"
  input: "ReadVariableOp_448"
  input: "mul_576"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_306"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_106"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_449"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_306"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_307"
  op: "AssignVariableOp"
  input: "Mul_572/ReadVariableOp/Enter"
  input: "add_358"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_450"
  op: "ReadVariableOp"
  input: "Mul_572/ReadVariableOp/Enter"
  input: "^AssignVariableOp_307"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_308"
  op: "AssignVariableOp"
  input: "Mul_574/ReadVariableOp/Enter"
  input: "add_359"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_451"
  op: "ReadVariableOp"
  input: "Mul_574/ReadVariableOp/Enter"
  input: "^AssignVariableOp_308"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_577/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_577/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_577/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_577/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_577"
  op: "Mul"
  input: "Mul_577/x"
  input: "Mul_577/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_578/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_578"
  op: "Mul"
  input: "Mul_578/x"
  input: "CrossReplicaSum_103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_361"
  op: "AddV2"
  input: "Mul_577"
  input: "Mul_578"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_579/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_579/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_579/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_579/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_579"
  op: "Mul"
  input: "Mul_579/x"
  input: "Mul_579/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_103"
  op: "Square"
  input: "CrossReplicaSum_103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_580/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_580"
  op: "Mul"
  input: "Mul_580/x"
  input: "Square_103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_362"
  op: "AddV2"
  input: "Mul_579"
  input: "Mul_580"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_103"
  op: "Sqrt"
  input: "add_362"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_363/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_363"
  op: "AddV2"
  input: "Sqrt_103"
  input: "add_363/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_106"
  op: "RealDiv"
  input: "add_361"
  input: "add_363"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_452"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_581/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_581"
  op: "Mul"
  input: "mul_581/x"
  input: "ReadVariableOp_452"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_364"
  op: "AddV2"
  input: "truediv_106"
  input: "mul_581"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_582"
  op: "Mul"
  input: "mul_21"
  input: "add_364"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_453"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_107"
  op: "Sub"
  input: "ReadVariableOp_453"
  input: "mul_582"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_309"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_107"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_454"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_309"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_310"
  op: "AssignVariableOp"
  input: "Mul_577/ReadVariableOp/Enter"
  input: "add_361"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_455"
  op: "ReadVariableOp"
  input: "Mul_577/ReadVariableOp/Enter"
  input: "^AssignVariableOp_310"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_311"
  op: "AssignVariableOp"
  input: "Mul_579/ReadVariableOp/Enter"
  input: "add_362"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_456"
  op: "ReadVariableOp"
  input: "Mul_579/ReadVariableOp/Enter"
  input: "^AssignVariableOp_311"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_583/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_583/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_583/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_583/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_583"
  op: "Mul"
  input: "Mul_583/x"
  input: "Mul_583/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_584/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_584"
  op: "Mul"
  input: "Mul_584/x"
  input: "CrossReplicaSum_104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_365"
  op: "AddV2"
  input: "Mul_583"
  input: "Mul_584"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_585/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_585/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_585/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_585/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_585"
  op: "Mul"
  input: "Mul_585/x"
  input: "Mul_585/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_104"
  op: "Square"
  input: "CrossReplicaSum_104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_586/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_586"
  op: "Mul"
  input: "Mul_586/x"
  input: "Square_104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_366"
  op: "AddV2"
  input: "Mul_585"
  input: "Mul_586"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_104"
  op: "Sqrt"
  input: "add_366"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_367/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_367"
  op: "AddV2"
  input: "Sqrt_104"
  input: "add_367/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_107"
  op: "RealDiv"
  input: "add_365"
  input: "add_367"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_587"
  op: "Mul"
  input: "mul_21"
  input: "truediv_107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_457"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_108"
  op: "Sub"
  input: "ReadVariableOp_457"
  input: "mul_587"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_312"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_108"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_458"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_312"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_313"
  op: "AssignVariableOp"
  input: "Mul_583/ReadVariableOp/Enter"
  input: "add_365"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_459"
  op: "ReadVariableOp"
  input: "Mul_583/ReadVariableOp/Enter"
  input: "^AssignVariableOp_313"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_314"
  op: "AssignVariableOp"
  input: "Mul_585/ReadVariableOp/Enter"
  input: "add_366"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_460"
  op: "ReadVariableOp"
  input: "Mul_585/ReadVariableOp/Enter"
  input: "^AssignVariableOp_314"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_588/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_588/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_588/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_588/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_588"
  op: "Mul"
  input: "Mul_588/x"
  input: "Mul_588/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_589/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_589"
  op: "Mul"
  input: "Mul_589/x"
  input: "CrossReplicaSum_105"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_368"
  op: "AddV2"
  input: "Mul_588"
  input: "Mul_589"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_590/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_590/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_590/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_590/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_590"
  op: "Mul"
  input: "Mul_590/x"
  input: "Mul_590/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_105"
  op: "Square"
  input: "CrossReplicaSum_105"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_591/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_591"
  op: "Mul"
  input: "Mul_591/x"
  input: "Square_105"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_369"
  op: "AddV2"
  input: "Mul_590"
  input: "Mul_591"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_105"
  op: "Sqrt"
  input: "add_369"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_370/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_370"
  op: "AddV2"
  input: "Sqrt_105"
  input: "add_370/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_108"
  op: "RealDiv"
  input: "add_368"
  input: "add_370"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_461"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_592/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_592"
  op: "Mul"
  input: "mul_592/x"
  input: "ReadVariableOp_461"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_371"
  op: "AddV2"
  input: "truediv_108"
  input: "mul_592"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_593"
  op: "Mul"
  input: "mul_21"
  input: "add_371"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_462"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_109"
  op: "Sub"
  input: "ReadVariableOp_462"
  input: "mul_593"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_315"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_109"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_463"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_315"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_316"
  op: "AssignVariableOp"
  input: "Mul_588/ReadVariableOp/Enter"
  input: "add_368"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_464"
  op: "ReadVariableOp"
  input: "Mul_588/ReadVariableOp/Enter"
  input: "^AssignVariableOp_316"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_317"
  op: "AssignVariableOp"
  input: "Mul_590/ReadVariableOp/Enter"
  input: "add_369"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_465"
  op: "ReadVariableOp"
  input: "Mul_590/ReadVariableOp/Enter"
  input: "^AssignVariableOp_317"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_594/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_594/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_594/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_594/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_594"
  op: "Mul"
  input: "Mul_594/x"
  input: "Mul_594/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_595/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_595"
  op: "Mul"
  input: "Mul_595/x"
  input: "CrossReplicaSum_106"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_372"
  op: "AddV2"
  input: "Mul_594"
  input: "Mul_595"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_596/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_596/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_596/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_596/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_596"
  op: "Mul"
  input: "Mul_596/x"
  input: "Mul_596/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_106"
  op: "Square"
  input: "CrossReplicaSum_106"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_597/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_597"
  op: "Mul"
  input: "Mul_597/x"
  input: "Square_106"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_373"
  op: "AddV2"
  input: "Mul_596"
  input: "Mul_597"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_106"
  op: "Sqrt"
  input: "add_373"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_374/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_374"
  op: "AddV2"
  input: "Sqrt_106"
  input: "add_374/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_109"
  op: "RealDiv"
  input: "add_372"
  input: "add_374"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_598"
  op: "Mul"
  input: "mul_21"
  input: "truediv_109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_466"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_110"
  op: "Sub"
  input: "ReadVariableOp_466"
  input: "mul_598"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_318"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_110"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_467"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_318"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_319"
  op: "AssignVariableOp"
  input: "Mul_594/ReadVariableOp/Enter"
  input: "add_372"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_468"
  op: "ReadVariableOp"
  input: "Mul_594/ReadVariableOp/Enter"
  input: "^AssignVariableOp_319"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_320"
  op: "AssignVariableOp"
  input: "Mul_596/ReadVariableOp/Enter"
  input: "add_373"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_469"
  op: "ReadVariableOp"
  input: "Mul_596/ReadVariableOp/Enter"
  input: "^AssignVariableOp_320"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_599/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_599/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_599/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_599/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_599"
  op: "Mul"
  input: "Mul_599/x"
  input: "Mul_599/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_600/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_600"
  op: "Mul"
  input: "Mul_600/x"
  input: "CrossReplicaSum_107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_375"
  op: "AddV2"
  input: "Mul_599"
  input: "Mul_600"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_601/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_601/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_601/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_601/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_601"
  op: "Mul"
  input: "Mul_601/x"
  input: "Mul_601/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_107"
  op: "Square"
  input: "CrossReplicaSum_107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_602/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_602"
  op: "Mul"
  input: "Mul_602/x"
  input: "Square_107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_376"
  op: "AddV2"
  input: "Mul_601"
  input: "Mul_602"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_107"
  op: "Sqrt"
  input: "add_376"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_377/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_377"
  op: "AddV2"
  input: "Sqrt_107"
  input: "add_377/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_110"
  op: "RealDiv"
  input: "add_375"
  input: "add_377"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_470"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_603/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_603"
  op: "Mul"
  input: "mul_603/x"
  input: "ReadVariableOp_470"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_378"
  op: "AddV2"
  input: "truediv_110"
  input: "mul_603"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_604"
  op: "Mul"
  input: "mul_21"
  input: "add_378"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_471"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_111"
  op: "Sub"
  input: "ReadVariableOp_471"
  input: "mul_604"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_321"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_111"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_472"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_321"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_322"
  op: "AssignVariableOp"
  input: "Mul_599/ReadVariableOp/Enter"
  input: "add_375"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_473"
  op: "ReadVariableOp"
  input: "Mul_599/ReadVariableOp/Enter"
  input: "^AssignVariableOp_322"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_323"
  op: "AssignVariableOp"
  input: "Mul_601/ReadVariableOp/Enter"
  input: "add_376"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_474"
  op: "ReadVariableOp"
  input: "Mul_601/ReadVariableOp/Enter"
  input: "^AssignVariableOp_323"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_605/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_605/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_605/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_605/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_605"
  op: "Mul"
  input: "Mul_605/x"
  input: "Mul_605/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_606/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_606"
  op: "Mul"
  input: "Mul_606/x"
  input: "CrossReplicaSum_108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_379"
  op: "AddV2"
  input: "Mul_605"
  input: "Mul_606"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_607/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_607/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_607/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_607/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_607"
  op: "Mul"
  input: "Mul_607/x"
  input: "Mul_607/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_108"
  op: "Square"
  input: "CrossReplicaSum_108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_608/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_608"
  op: "Mul"
  input: "Mul_608/x"
  input: "Square_108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_380"
  op: "AddV2"
  input: "Mul_607"
  input: "Mul_608"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_108"
  op: "Sqrt"
  input: "add_380"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_381/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_381"
  op: "AddV2"
  input: "Sqrt_108"
  input: "add_381/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_111"
  op: "RealDiv"
  input: "add_379"
  input: "add_381"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_609"
  op: "Mul"
  input: "mul_21"
  input: "truediv_111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_475"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_112"
  op: "Sub"
  input: "ReadVariableOp_475"
  input: "mul_609"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_324"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_112"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_476"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_324"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_325"
  op: "AssignVariableOp"
  input: "Mul_605/ReadVariableOp/Enter"
  input: "add_379"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_477"
  op: "ReadVariableOp"
  input: "Mul_605/ReadVariableOp/Enter"
  input: "^AssignVariableOp_325"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_326"
  op: "AssignVariableOp"
  input: "Mul_607/ReadVariableOp/Enter"
  input: "add_380"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_478"
  op: "ReadVariableOp"
  input: "Mul_607/ReadVariableOp/Enter"
  input: "^AssignVariableOp_326"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_610/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_610/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_610/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_610/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_610"
  op: "Mul"
  input: "Mul_610/x"
  input: "Mul_610/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_611/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_611"
  op: "Mul"
  input: "Mul_611/x"
  input: "CrossReplicaSum_109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_382"
  op: "AddV2"
  input: "Mul_610"
  input: "Mul_611"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_612/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_612/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_612/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_612/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_612"
  op: "Mul"
  input: "Mul_612/x"
  input: "Mul_612/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_109"
  op: "Square"
  input: "CrossReplicaSum_109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_613/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_613"
  op: "Mul"
  input: "Mul_613/x"
  input: "Square_109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_383"
  op: "AddV2"
  input: "Mul_612"
  input: "Mul_613"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_109"
  op: "Sqrt"
  input: "add_383"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_384/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_384"
  op: "AddV2"
  input: "Sqrt_109"
  input: "add_384/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_112"
  op: "RealDiv"
  input: "add_382"
  input: "add_384"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_479"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_614/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_614"
  op: "Mul"
  input: "mul_614/x"
  input: "ReadVariableOp_479"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_385"
  op: "AddV2"
  input: "truediv_112"
  input: "mul_614"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_615"
  op: "Mul"
  input: "mul_21"
  input: "add_385"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_480"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_113"
  op: "Sub"
  input: "ReadVariableOp_480"
  input: "mul_615"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_327"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_113"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_481"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_327"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_328"
  op: "AssignVariableOp"
  input: "Mul_610/ReadVariableOp/Enter"
  input: "add_382"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_482"
  op: "ReadVariableOp"
  input: "Mul_610/ReadVariableOp/Enter"
  input: "^AssignVariableOp_328"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_329"
  op: "AssignVariableOp"
  input: "Mul_612/ReadVariableOp/Enter"
  input: "add_383"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_483"
  op: "ReadVariableOp"
  input: "Mul_612/ReadVariableOp/Enter"
  input: "^AssignVariableOp_329"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_616/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_616/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_616/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_616/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_616"
  op: "Mul"
  input: "Mul_616/x"
  input: "Mul_616/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_617/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_617"
  op: "Mul"
  input: "Mul_617/x"
  input: "CrossReplicaSum_110"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_386"
  op: "AddV2"
  input: "Mul_616"
  input: "Mul_617"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_618/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_618/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_618/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_618/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_618"
  op: "Mul"
  input: "Mul_618/x"
  input: "Mul_618/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_110"
  op: "Square"
  input: "CrossReplicaSum_110"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_619/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_619"
  op: "Mul"
  input: "Mul_619/x"
  input: "Square_110"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_387"
  op: "AddV2"
  input: "Mul_618"
  input: "Mul_619"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_110"
  op: "Sqrt"
  input: "add_387"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_388/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_388"
  op: "AddV2"
  input: "Sqrt_110"
  input: "add_388/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_113"
  op: "RealDiv"
  input: "add_386"
  input: "add_388"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_620"
  op: "Mul"
  input: "mul_21"
  input: "truediv_113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_484"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_114"
  op: "Sub"
  input: "ReadVariableOp_484"
  input: "mul_620"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_330"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_114"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_485"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_330"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_331"
  op: "AssignVariableOp"
  input: "Mul_616/ReadVariableOp/Enter"
  input: "add_386"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_486"
  op: "ReadVariableOp"
  input: "Mul_616/ReadVariableOp/Enter"
  input: "^AssignVariableOp_331"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_332"
  op: "AssignVariableOp"
  input: "Mul_618/ReadVariableOp/Enter"
  input: "add_387"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_487"
  op: "ReadVariableOp"
  input: "Mul_618/ReadVariableOp/Enter"
  input: "^AssignVariableOp_332"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_621/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_621/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_621/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_621/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_621"
  op: "Mul"
  input: "Mul_621/x"
  input: "Mul_621/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_622/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_622"
  op: "Mul"
  input: "Mul_622/x"
  input: "CrossReplicaSum_111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_389"
  op: "AddV2"
  input: "Mul_621"
  input: "Mul_622"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_623/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_623/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_623/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_623/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_623"
  op: "Mul"
  input: "Mul_623/x"
  input: "Mul_623/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_111"
  op: "Square"
  input: "CrossReplicaSum_111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_624/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_624"
  op: "Mul"
  input: "Mul_624/x"
  input: "Square_111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_390"
  op: "AddV2"
  input: "Mul_623"
  input: "Mul_624"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_111"
  op: "Sqrt"
  input: "add_390"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_391/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_391"
  op: "AddV2"
  input: "Sqrt_111"
  input: "add_391/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_114"
  op: "RealDiv"
  input: "add_389"
  input: "add_391"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_625"
  op: "Mul"
  input: "mul_21"
  input: "truediv_114"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_488"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_115"
  op: "Sub"
  input: "ReadVariableOp_488"
  input: "mul_625"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_333"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_115"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_489"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_333"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_334"
  op: "AssignVariableOp"
  input: "Mul_621/ReadVariableOp/Enter"
  input: "add_389"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_490"
  op: "ReadVariableOp"
  input: "Mul_621/ReadVariableOp/Enter"
  input: "^AssignVariableOp_334"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_335"
  op: "AssignVariableOp"
  input: "Mul_623/ReadVariableOp/Enter"
  input: "add_390"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_491"
  op: "ReadVariableOp"
  input: "Mul_623/ReadVariableOp/Enter"
  input: "^AssignVariableOp_335"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_626/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_626/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_626/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_626/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_626"
  op: "Mul"
  input: "Mul_626/x"
  input: "Mul_626/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_627/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_627"
  op: "Mul"
  input: "Mul_627/x"
  input: "CrossReplicaSum_112"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_392"
  op: "AddV2"
  input: "Mul_626"
  input: "Mul_627"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_628/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_628/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_628/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_628/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_628"
  op: "Mul"
  input: "Mul_628/x"
  input: "Mul_628/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_112"
  op: "Square"
  input: "CrossReplicaSum_112"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_629/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_629"
  op: "Mul"
  input: "Mul_629/x"
  input: "Square_112"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_393"
  op: "AddV2"
  input: "Mul_628"
  input: "Mul_629"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_112"
  op: "Sqrt"
  input: "add_393"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_394/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_394"
  op: "AddV2"
  input: "Sqrt_112"
  input: "add_394/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_115"
  op: "RealDiv"
  input: "add_392"
  input: "add_394"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_630"
  op: "Mul"
  input: "mul_21"
  input: "truediv_115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_492"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_116"
  op: "Sub"
  input: "ReadVariableOp_492"
  input: "mul_630"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_336"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_116"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_493"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_336"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_337"
  op: "AssignVariableOp"
  input: "Mul_626/ReadVariableOp/Enter"
  input: "add_392"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_494"
  op: "ReadVariableOp"
  input: "Mul_626/ReadVariableOp/Enter"
  input: "^AssignVariableOp_337"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_338"
  op: "AssignVariableOp"
  input: "Mul_628/ReadVariableOp/Enter"
  input: "add_393"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_495"
  op: "ReadVariableOp"
  input: "Mul_628/ReadVariableOp/Enter"
  input: "^AssignVariableOp_338"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_631/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_631/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_631/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_631/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_631"
  op: "Mul"
  input: "Mul_631/x"
  input: "Mul_631/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_632/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_632"
  op: "Mul"
  input: "Mul_632/x"
  input: "CrossReplicaSum_113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_395"
  op: "AddV2"
  input: "Mul_631"
  input: "Mul_632"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_633/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_633/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_633/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_633/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_633"
  op: "Mul"
  input: "Mul_633/x"
  input: "Mul_633/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_113"
  op: "Square"
  input: "CrossReplicaSum_113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_634/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_634"
  op: "Mul"
  input: "Mul_634/x"
  input: "Square_113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_396"
  op: "AddV2"
  input: "Mul_633"
  input: "Mul_634"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_113"
  op: "Sqrt"
  input: "add_396"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_397/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_397"
  op: "AddV2"
  input: "Sqrt_113"
  input: "add_397/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_116"
  op: "RealDiv"
  input: "add_395"
  input: "add_397"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_496"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_635/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_635"
  op: "Mul"
  input: "mul_635/x"
  input: "ReadVariableOp_496"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_398"
  op: "AddV2"
  input: "truediv_116"
  input: "mul_635"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_636"
  op: "Mul"
  input: "mul_21"
  input: "add_398"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_497"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_117"
  op: "Sub"
  input: "ReadVariableOp_497"
  input: "mul_636"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_339"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_117"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_498"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_339"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_340"
  op: "AssignVariableOp"
  input: "Mul_631/ReadVariableOp/Enter"
  input: "add_395"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_499"
  op: "ReadVariableOp"
  input: "Mul_631/ReadVariableOp/Enter"
  input: "^AssignVariableOp_340"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_341"
  op: "AssignVariableOp"
  input: "Mul_633/ReadVariableOp/Enter"
  input: "add_396"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_500"
  op: "ReadVariableOp"
  input: "Mul_633/ReadVariableOp/Enter"
  input: "^AssignVariableOp_341"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_637/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_637/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_637/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_637/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_637"
  op: "Mul"
  input: "Mul_637/x"
  input: "Mul_637/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_638/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_638"
  op: "Mul"
  input: "Mul_638/x"
  input: "CrossReplicaSum_114"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_399"
  op: "AddV2"
  input: "Mul_637"
  input: "Mul_638"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_639/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_639/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_639/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_639/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_639"
  op: "Mul"
  input: "Mul_639/x"
  input: "Mul_639/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_114"
  op: "Square"
  input: "CrossReplicaSum_114"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_640/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_640"
  op: "Mul"
  input: "Mul_640/x"
  input: "Square_114"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_400"
  op: "AddV2"
  input: "Mul_639"
  input: "Mul_640"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_114"
  op: "Sqrt"
  input: "add_400"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_401/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_401"
  op: "AddV2"
  input: "Sqrt_114"
  input: "add_401/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_117"
  op: "RealDiv"
  input: "add_399"
  input: "add_401"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_641"
  op: "Mul"
  input: "mul_21"
  input: "truediv_117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_501"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_118"
  op: "Sub"
  input: "ReadVariableOp_501"
  input: "mul_641"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_342"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_118"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_502"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_342"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_343"
  op: "AssignVariableOp"
  input: "Mul_637/ReadVariableOp/Enter"
  input: "add_399"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_503"
  op: "ReadVariableOp"
  input: "Mul_637/ReadVariableOp/Enter"
  input: "^AssignVariableOp_343"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_344"
  op: "AssignVariableOp"
  input: "Mul_639/ReadVariableOp/Enter"
  input: "add_400"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_504"
  op: "ReadVariableOp"
  input: "Mul_639/ReadVariableOp/Enter"
  input: "^AssignVariableOp_344"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_642/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_642/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_642/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_642/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_642"
  op: "Mul"
  input: "Mul_642/x"
  input: "Mul_642/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_643/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_643"
  op: "Mul"
  input: "Mul_643/x"
  input: "CrossReplicaSum_115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_402"
  op: "AddV2"
  input: "Mul_642"
  input: "Mul_643"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_644/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_644/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_644/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_644/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_644"
  op: "Mul"
  input: "Mul_644/x"
  input: "Mul_644/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_115"
  op: "Square"
  input: "CrossReplicaSum_115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_645/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_645"
  op: "Mul"
  input: "Mul_645/x"
  input: "Square_115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_403"
  op: "AddV2"
  input: "Mul_644"
  input: "Mul_645"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_115"
  op: "Sqrt"
  input: "add_403"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_404/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_404"
  op: "AddV2"
  input: "Sqrt_115"
  input: "add_404/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_118"
  op: "RealDiv"
  input: "add_402"
  input: "add_404"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_505"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_646/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_646"
  op: "Mul"
  input: "mul_646/x"
  input: "ReadVariableOp_505"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_405"
  op: "AddV2"
  input: "truediv_118"
  input: "mul_646"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_647"
  op: "Mul"
  input: "mul_21"
  input: "add_405"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_506"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_119"
  op: "Sub"
  input: "ReadVariableOp_506"
  input: "mul_647"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_345"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_119"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_507"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_345"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_346"
  op: "AssignVariableOp"
  input: "Mul_642/ReadVariableOp/Enter"
  input: "add_402"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_508"
  op: "ReadVariableOp"
  input: "Mul_642/ReadVariableOp/Enter"
  input: "^AssignVariableOp_346"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_347"
  op: "AssignVariableOp"
  input: "Mul_644/ReadVariableOp/Enter"
  input: "add_403"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_509"
  op: "ReadVariableOp"
  input: "Mul_644/ReadVariableOp/Enter"
  input: "^AssignVariableOp_347"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_m"
  input: "generator/encoder/layer_6/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_v"
  input: "generator/encoder/layer_6/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_648/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_648/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_648/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_648/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_648"
  op: "Mul"
  input: "Mul_648/x"
  input: "Mul_648/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_649/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_649"
  op: "Mul"
  input: "Mul_649/x"
  input: "CrossReplicaSum_116"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_406"
  op: "AddV2"
  input: "Mul_648"
  input: "Mul_649"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_650/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_650/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_650/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_650/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_650"
  op: "Mul"
  input: "Mul_650/x"
  input: "Mul_650/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_116"
  op: "Square"
  input: "CrossReplicaSum_116"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_651/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_651"
  op: "Mul"
  input: "Mul_651/x"
  input: "Square_116"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_407"
  op: "AddV2"
  input: "Mul_650"
  input: "Mul_651"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_116"
  op: "Sqrt"
  input: "add_407"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_408/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_408"
  op: "AddV2"
  input: "Sqrt_116"
  input: "add_408/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_119"
  op: "RealDiv"
  input: "add_406"
  input: "add_408"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_652"
  op: "Mul"
  input: "mul_21"
  input: "truediv_119"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_510"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_120"
  op: "Sub"
  input: "ReadVariableOp_510"
  input: "mul_652"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_348"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_120"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_511"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_348"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_349"
  op: "AssignVariableOp"
  input: "Mul_648/ReadVariableOp/Enter"
  input: "add_406"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_512"
  op: "ReadVariableOp"
  input: "Mul_648/ReadVariableOp/Enter"
  input: "^AssignVariableOp_349"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_350"
  op: "AssignVariableOp"
  input: "Mul_650/ReadVariableOp/Enter"
  input: "add_407"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_513"
  op: "ReadVariableOp"
  input: "Mul_650/ReadVariableOp/Enter"
  input: "^AssignVariableOp_350"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_653/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_653/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_653/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_653/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_653"
  op: "Mul"
  input: "Mul_653/x"
  input: "Mul_653/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_654/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_654"
  op: "Mul"
  input: "Mul_654/x"
  input: "CrossReplicaSum_117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_409"
  op: "AddV2"
  input: "Mul_653"
  input: "Mul_654"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_655/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_655/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_655/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_655/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_655"
  op: "Mul"
  input: "Mul_655/x"
  input: "Mul_655/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_117"
  op: "Square"
  input: "CrossReplicaSum_117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_656/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_656"
  op: "Mul"
  input: "Mul_656/x"
  input: "Square_117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_410"
  op: "AddV2"
  input: "Mul_655"
  input: "Mul_656"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_117"
  op: "Sqrt"
  input: "add_410"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_411/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_411"
  op: "AddV2"
  input: "Sqrt_117"
  input: "add_411/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_120"
  op: "RealDiv"
  input: "add_409"
  input: "add_411"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_657"
  op: "Mul"
  input: "mul_21"
  input: "truediv_120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_514"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_121"
  op: "Sub"
  input: "ReadVariableOp_514"
  input: "mul_657"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_351"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_121"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_515"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_351"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_352"
  op: "AssignVariableOp"
  input: "Mul_653/ReadVariableOp/Enter"
  input: "add_409"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_516"
  op: "ReadVariableOp"
  input: "Mul_653/ReadVariableOp/Enter"
  input: "^AssignVariableOp_352"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_353"
  op: "AssignVariableOp"
  input: "Mul_655/ReadVariableOp/Enter"
  input: "add_410"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_517"
  op: "ReadVariableOp"
  input: "Mul_655/ReadVariableOp/Enter"
  input: "^AssignVariableOp_353"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_658/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_658/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_658/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_658/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_658"
  op: "Mul"
  input: "Mul_658/x"
  input: "Mul_658/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_659/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_659"
  op: "Mul"
  input: "Mul_659/x"
  input: "CrossReplicaSum_118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_412"
  op: "AddV2"
  input: "Mul_658"
  input: "Mul_659"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_660/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_660/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_660/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_660/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_660"
  op: "Mul"
  input: "Mul_660/x"
  input: "Mul_660/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_118"
  op: "Square"
  input: "CrossReplicaSum_118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_661/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_661"
  op: "Mul"
  input: "Mul_661/x"
  input: "Square_118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_413"
  op: "AddV2"
  input: "Mul_660"
  input: "Mul_661"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_118"
  op: "Sqrt"
  input: "add_413"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_414/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_414"
  op: "AddV2"
  input: "Sqrt_118"
  input: "add_414/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_121"
  op: "RealDiv"
  input: "add_412"
  input: "add_414"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_662"
  op: "Mul"
  input: "mul_21"
  input: "truediv_121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_518"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_122"
  op: "Sub"
  input: "ReadVariableOp_518"
  input: "mul_662"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_354"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_122"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_519"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_354"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_355"
  op: "AssignVariableOp"
  input: "Mul_658/ReadVariableOp/Enter"
  input: "add_412"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_520"
  op: "ReadVariableOp"
  input: "Mul_658/ReadVariableOp/Enter"
  input: "^AssignVariableOp_355"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_356"
  op: "AssignVariableOp"
  input: "Mul_660/ReadVariableOp/Enter"
  input: "add_413"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_521"
  op: "ReadVariableOp"
  input: "Mul_660/ReadVariableOp/Enter"
  input: "^AssignVariableOp_356"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_663/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_663/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_663/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_663/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_663"
  op: "Mul"
  input: "Mul_663/x"
  input: "Mul_663/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_664/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_664"
  op: "Mul"
  input: "Mul_664/x"
  input: "CrossReplicaSum_119"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_415"
  op: "AddV2"
  input: "Mul_663"
  input: "Mul_664"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_665/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_665/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_665/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_665/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_665"
  op: "Mul"
  input: "Mul_665/x"
  input: "Mul_665/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_119"
  op: "Square"
  input: "CrossReplicaSum_119"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_666/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_666"
  op: "Mul"
  input: "Mul_666/x"
  input: "Square_119"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_416"
  op: "AddV2"
  input: "Mul_665"
  input: "Mul_666"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_119"
  op: "Sqrt"
  input: "add_416"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_417/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_417"
  op: "AddV2"
  input: "Sqrt_119"
  input: "add_417/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_122"
  op: "RealDiv"
  input: "add_415"
  input: "add_417"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_522"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_667/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_667"
  op: "Mul"
  input: "mul_667/x"
  input: "ReadVariableOp_522"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_418"
  op: "AddV2"
  input: "truediv_122"
  input: "mul_667"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_668"
  op: "Mul"
  input: "mul_21"
  input: "add_418"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_523"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_123"
  op: "Sub"
  input: "ReadVariableOp_523"
  input: "mul_668"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_357"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_123"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_524"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_357"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_358"
  op: "AssignVariableOp"
  input: "Mul_663/ReadVariableOp/Enter"
  input: "add_415"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_525"
  op: "ReadVariableOp"
  input: "Mul_663/ReadVariableOp/Enter"
  input: "^AssignVariableOp_358"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_359"
  op: "AssignVariableOp"
  input: "Mul_665/ReadVariableOp/Enter"
  input: "add_416"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_526"
  op: "ReadVariableOp"
  input: "Mul_665/ReadVariableOp/Enter"
  input: "^AssignVariableOp_359"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_669/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_669/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_669/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_669/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_669"
  op: "Mul"
  input: "Mul_669/x"
  input: "Mul_669/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_670/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_670"
  op: "Mul"
  input: "Mul_670/x"
  input: "CrossReplicaSum_120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_419"
  op: "AddV2"
  input: "Mul_669"
  input: "Mul_670"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_671/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_671/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_671/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_671/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_671"
  op: "Mul"
  input: "Mul_671/x"
  input: "Mul_671/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_120"
  op: "Square"
  input: "CrossReplicaSum_120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_672/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_672"
  op: "Mul"
  input: "Mul_672/x"
  input: "Square_120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_420"
  op: "AddV2"
  input: "Mul_671"
  input: "Mul_672"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_120"
  op: "Sqrt"
  input: "add_420"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_421/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_421"
  op: "AddV2"
  input: "Sqrt_120"
  input: "add_421/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_123"
  op: "RealDiv"
  input: "add_419"
  input: "add_421"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_673"
  op: "Mul"
  input: "mul_21"
  input: "truediv_123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_527"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_124"
  op: "Sub"
  input: "ReadVariableOp_527"
  input: "mul_673"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_360"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_124"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_528"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_360"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_361"
  op: "AssignVariableOp"
  input: "Mul_669/ReadVariableOp/Enter"
  input: "add_419"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_529"
  op: "ReadVariableOp"
  input: "Mul_669/ReadVariableOp/Enter"
  input: "^AssignVariableOp_361"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_362"
  op: "AssignVariableOp"
  input: "Mul_671/ReadVariableOp/Enter"
  input: "add_420"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_530"
  op: "ReadVariableOp"
  input: "Mul_671/ReadVariableOp/Enter"
  input: "^AssignVariableOp_362"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_674/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_674/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_674/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_674/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_674"
  op: "Mul"
  input: "Mul_674/x"
  input: "Mul_674/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_675/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_675"
  op: "Mul"
  input: "Mul_675/x"
  input: "CrossReplicaSum_121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_422"
  op: "AddV2"
  input: "Mul_674"
  input: "Mul_675"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_676/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_676/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_676/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_676/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_676"
  op: "Mul"
  input: "Mul_676/x"
  input: "Mul_676/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_121"
  op: "Square"
  input: "CrossReplicaSum_121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_677/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_677"
  op: "Mul"
  input: "Mul_677/x"
  input: "Square_121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_423"
  op: "AddV2"
  input: "Mul_676"
  input: "Mul_677"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_121"
  op: "Sqrt"
  input: "add_423"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_424/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_424"
  op: "AddV2"
  input: "Sqrt_121"
  input: "add_424/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_124"
  op: "RealDiv"
  input: "add_422"
  input: "add_424"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_531"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_678/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_678"
  op: "Mul"
  input: "mul_678/x"
  input: "ReadVariableOp_531"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_425"
  op: "AddV2"
  input: "truediv_124"
  input: "mul_678"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_679"
  op: "Mul"
  input: "mul_21"
  input: "add_425"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_532"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_125"
  op: "Sub"
  input: "ReadVariableOp_532"
  input: "mul_679"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_363"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_125"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_533"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_363"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_364"
  op: "AssignVariableOp"
  input: "Mul_674/ReadVariableOp/Enter"
  input: "add_422"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_534"
  op: "ReadVariableOp"
  input: "Mul_674/ReadVariableOp/Enter"
  input: "^AssignVariableOp_364"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_365"
  op: "AssignVariableOp"
  input: "Mul_676/ReadVariableOp/Enter"
  input: "add_423"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_535"
  op: "ReadVariableOp"
  input: "Mul_676/ReadVariableOp/Enter"
  input: "^AssignVariableOp_365"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_680/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_680/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_680/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_680/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_680"
  op: "Mul"
  input: "Mul_680/x"
  input: "Mul_680/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_681/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_681"
  op: "Mul"
  input: "Mul_681/x"
  input: "CrossReplicaSum_122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_426"
  op: "AddV2"
  input: "Mul_680"
  input: "Mul_681"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_682/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_682/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_682/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_682/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_682"
  op: "Mul"
  input: "Mul_682/x"
  input: "Mul_682/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_122"
  op: "Square"
  input: "CrossReplicaSum_122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_683/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_683"
  op: "Mul"
  input: "Mul_683/x"
  input: "Square_122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_427"
  op: "AddV2"
  input: "Mul_682"
  input: "Mul_683"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_122"
  op: "Sqrt"
  input: "add_427"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_428/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_428"
  op: "AddV2"
  input: "Sqrt_122"
  input: "add_428/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_125"
  op: "RealDiv"
  input: "add_426"
  input: "add_428"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_684"
  op: "Mul"
  input: "mul_21"
  input: "truediv_125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_536"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_126"
  op: "Sub"
  input: "ReadVariableOp_536"
  input: "mul_684"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_366"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_126"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_537"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_366"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_367"
  op: "AssignVariableOp"
  input: "Mul_680/ReadVariableOp/Enter"
  input: "add_426"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_538"
  op: "ReadVariableOp"
  input: "Mul_680/ReadVariableOp/Enter"
  input: "^AssignVariableOp_367"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_368"
  op: "AssignVariableOp"
  input: "Mul_682/ReadVariableOp/Enter"
  input: "add_427"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_539"
  op: "ReadVariableOp"
  input: "Mul_682/ReadVariableOp/Enter"
  input: "^AssignVariableOp_368"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_685/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_685/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_685/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_685/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_685"
  op: "Mul"
  input: "Mul_685/x"
  input: "Mul_685/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_686/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_686"
  op: "Mul"
  input: "Mul_686/x"
  input: "CrossReplicaSum_123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_429"
  op: "AddV2"
  input: "Mul_685"
  input: "Mul_686"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_687/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_687/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_687/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_687/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_687"
  op: "Mul"
  input: "Mul_687/x"
  input: "Mul_687/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_123"
  op: "Square"
  input: "CrossReplicaSum_123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_688/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_688"
  op: "Mul"
  input: "Mul_688/x"
  input: "Square_123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_430"
  op: "AddV2"
  input: "Mul_687"
  input: "Mul_688"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_123"
  op: "Sqrt"
  input: "add_430"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_431/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_431"
  op: "AddV2"
  input: "Sqrt_123"
  input: "add_431/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_126"
  op: "RealDiv"
  input: "add_429"
  input: "add_431"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_540"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_689/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_689"
  op: "Mul"
  input: "mul_689/x"
  input: "ReadVariableOp_540"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_432"
  op: "AddV2"
  input: "truediv_126"
  input: "mul_689"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_690"
  op: "Mul"
  input: "mul_21"
  input: "add_432"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_541"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_127"
  op: "Sub"
  input: "ReadVariableOp_541"
  input: "mul_690"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_369"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_127"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_542"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_369"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_370"
  op: "AssignVariableOp"
  input: "Mul_685/ReadVariableOp/Enter"
  input: "add_429"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_543"
  op: "ReadVariableOp"
  input: "Mul_685/ReadVariableOp/Enter"
  input: "^AssignVariableOp_370"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_371"
  op: "AssignVariableOp"
  input: "Mul_687/ReadVariableOp/Enter"
  input: "add_430"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_544"
  op: "ReadVariableOp"
  input: "Mul_687/ReadVariableOp/Enter"
  input: "^AssignVariableOp_371"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_691/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_691/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_691/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_691/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_691"
  op: "Mul"
  input: "Mul_691/x"
  input: "Mul_691/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_692/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_692"
  op: "Mul"
  input: "Mul_692/x"
  input: "CrossReplicaSum_124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_433"
  op: "AddV2"
  input: "Mul_691"
  input: "Mul_692"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_693/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_693/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_693/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_693/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_693"
  op: "Mul"
  input: "Mul_693/x"
  input: "Mul_693/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_124"
  op: "Square"
  input: "CrossReplicaSum_124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_694/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_694"
  op: "Mul"
  input: "Mul_694/x"
  input: "Square_124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_434"
  op: "AddV2"
  input: "Mul_693"
  input: "Mul_694"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_124"
  op: "Sqrt"
  input: "add_434"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_435/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_435"
  op: "AddV2"
  input: "Sqrt_124"
  input: "add_435/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_127"
  op: "RealDiv"
  input: "add_433"
  input: "add_435"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_695"
  op: "Mul"
  input: "mul_21"
  input: "truediv_127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_545"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_128"
  op: "Sub"
  input: "ReadVariableOp_545"
  input: "mul_695"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_372"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_128"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_546"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_372"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_373"
  op: "AssignVariableOp"
  input: "Mul_691/ReadVariableOp/Enter"
  input: "add_433"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_547"
  op: "ReadVariableOp"
  input: "Mul_691/ReadVariableOp/Enter"
  input: "^AssignVariableOp_373"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_374"
  op: "AssignVariableOp"
  input: "Mul_693/ReadVariableOp/Enter"
  input: "add_434"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_548"
  op: "ReadVariableOp"
  input: "Mul_693/ReadVariableOp/Enter"
  input: "^AssignVariableOp_374"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_696/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_696/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_696/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_696/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_696"
  op: "Mul"
  input: "Mul_696/x"
  input: "Mul_696/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_697/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_697"
  op: "Mul"
  input: "Mul_697/x"
  input: "CrossReplicaSum_125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_436"
  op: "AddV2"
  input: "Mul_696"
  input: "Mul_697"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_698/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_698/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_698/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_698/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_698"
  op: "Mul"
  input: "Mul_698/x"
  input: "Mul_698/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_125"
  op: "Square"
  input: "CrossReplicaSum_125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_699/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_699"
  op: "Mul"
  input: "Mul_699/x"
  input: "Square_125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_437"
  op: "AddV2"
  input: "Mul_698"
  input: "Mul_699"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_125"
  op: "Sqrt"
  input: "add_437"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_438/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_438"
  op: "AddV2"
  input: "Sqrt_125"
  input: "add_438/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_128"
  op: "RealDiv"
  input: "add_436"
  input: "add_438"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_549"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_700/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_700"
  op: "Mul"
  input: "mul_700/x"
  input: "ReadVariableOp_549"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_439"
  op: "AddV2"
  input: "truediv_128"
  input: "mul_700"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_701"
  op: "Mul"
  input: "mul_21"
  input: "add_439"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_550"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_129"
  op: "Sub"
  input: "ReadVariableOp_550"
  input: "mul_701"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_375"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_129"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_551"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_375"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_376"
  op: "AssignVariableOp"
  input: "Mul_696/ReadVariableOp/Enter"
  input: "add_436"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_552"
  op: "ReadVariableOp"
  input: "Mul_696/ReadVariableOp/Enter"
  input: "^AssignVariableOp_376"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_377"
  op: "AssignVariableOp"
  input: "Mul_698/ReadVariableOp/Enter"
  input: "add_437"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_553"
  op: "ReadVariableOp"
  input: "Mul_698/ReadVariableOp/Enter"
  input: "^AssignVariableOp_377"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_702/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_702/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_702/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_702/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_702"
  op: "Mul"
  input: "Mul_702/x"
  input: "Mul_702/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_703/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_703"
  op: "Mul"
  input: "Mul_703/x"
  input: "CrossReplicaSum_126"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_440"
  op: "AddV2"
  input: "Mul_702"
  input: "Mul_703"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_704/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_704/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_704/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_704/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_704"
  op: "Mul"
  input: "Mul_704/x"
  input: "Mul_704/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_126"
  op: "Square"
  input: "CrossReplicaSum_126"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_705/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_705"
  op: "Mul"
  input: "Mul_705/x"
  input: "Square_126"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_441"
  op: "AddV2"
  input: "Mul_704"
  input: "Mul_705"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_126"
  op: "Sqrt"
  input: "add_441"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_442/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_442"
  op: "AddV2"
  input: "Sqrt_126"
  input: "add_442/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_129"
  op: "RealDiv"
  input: "add_440"
  input: "add_442"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_706"
  op: "Mul"
  input: "mul_21"
  input: "truediv_129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_554"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_130"
  op: "Sub"
  input: "ReadVariableOp_554"
  input: "mul_706"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_378"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_130"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_555"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_378"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_379"
  op: "AssignVariableOp"
  input: "Mul_702/ReadVariableOp/Enter"
  input: "add_440"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_556"
  op: "ReadVariableOp"
  input: "Mul_702/ReadVariableOp/Enter"
  input: "^AssignVariableOp_379"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_380"
  op: "AssignVariableOp"
  input: "Mul_704/ReadVariableOp/Enter"
  input: "add_441"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_557"
  op: "ReadVariableOp"
  input: "Mul_704/ReadVariableOp/Enter"
  input: "^AssignVariableOp_380"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_707/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_707/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_707/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_707/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_707"
  op: "Mul"
  input: "Mul_707/x"
  input: "Mul_707/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_708/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_708"
  op: "Mul"
  input: "Mul_708/x"
  input: "CrossReplicaSum_127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_443"
  op: "AddV2"
  input: "Mul_707"
  input: "Mul_708"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_709/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_709/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_709/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_709/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_709"
  op: "Mul"
  input: "Mul_709/x"
  input: "Mul_709/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_127"
  op: "Square"
  input: "CrossReplicaSum_127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_710/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_710"
  op: "Mul"
  input: "Mul_710/x"
  input: "Square_127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_444"
  op: "AddV2"
  input: "Mul_709"
  input: "Mul_710"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_127"
  op: "Sqrt"
  input: "add_444"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_445/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_445"
  op: "AddV2"
  input: "Sqrt_127"
  input: "add_445/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_130"
  op: "RealDiv"
  input: "add_443"
  input: "add_445"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_711"
  op: "Mul"
  input: "mul_21"
  input: "truediv_130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_558"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_131"
  op: "Sub"
  input: "ReadVariableOp_558"
  input: "mul_711"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_381"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_131"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_559"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_381"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_382"
  op: "AssignVariableOp"
  input: "Mul_707/ReadVariableOp/Enter"
  input: "add_443"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_560"
  op: "ReadVariableOp"
  input: "Mul_707/ReadVariableOp/Enter"
  input: "^AssignVariableOp_382"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_383"
  op: "AssignVariableOp"
  input: "Mul_709/ReadVariableOp/Enter"
  input: "add_444"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_561"
  op: "ReadVariableOp"
  input: "Mul_709/ReadVariableOp/Enter"
  input: "^AssignVariableOp_383"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_712/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_712/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_712/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_712/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_712"
  op: "Mul"
  input: "Mul_712/x"
  input: "Mul_712/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_713/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_713"
  op: "Mul"
  input: "Mul_713/x"
  input: "CrossReplicaSum_128"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_446"
  op: "AddV2"
  input: "Mul_712"
  input: "Mul_713"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_714/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_714/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_714/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_714/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_714"
  op: "Mul"
  input: "Mul_714/x"
  input: "Mul_714/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_128"
  op: "Square"
  input: "CrossReplicaSum_128"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_715/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_715"
  op: "Mul"
  input: "Mul_715/x"
  input: "Square_128"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_447"
  op: "AddV2"
  input: "Mul_714"
  input: "Mul_715"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_128"
  op: "Sqrt"
  input: "add_447"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_448/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_448"
  op: "AddV2"
  input: "Sqrt_128"
  input: "add_448/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_131"
  op: "RealDiv"
  input: "add_446"
  input: "add_448"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_716"
  op: "Mul"
  input: "mul_21"
  input: "truediv_131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_562"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_132"
  op: "Sub"
  input: "ReadVariableOp_562"
  input: "mul_716"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_384"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_132"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_563"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_384"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_385"
  op: "AssignVariableOp"
  input: "Mul_712/ReadVariableOp/Enter"
  input: "add_446"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_564"
  op: "ReadVariableOp"
  input: "Mul_712/ReadVariableOp/Enter"
  input: "^AssignVariableOp_385"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_386"
  op: "AssignVariableOp"
  input: "Mul_714/ReadVariableOp/Enter"
  input: "add_447"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_565"
  op: "ReadVariableOp"
  input: "Mul_714/ReadVariableOp/Enter"
  input: "^AssignVariableOp_386"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_717/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_717/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_717/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_717/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_717"
  op: "Mul"
  input: "Mul_717/x"
  input: "Mul_717/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_718/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_718"
  op: "Mul"
  input: "Mul_718/x"
  input: "CrossReplicaSum_129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_449"
  op: "AddV2"
  input: "Mul_717"
  input: "Mul_718"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_719/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_719/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_719/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_719/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_719"
  op: "Mul"
  input: "Mul_719/x"
  input: "Mul_719/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_129"
  op: "Square"
  input: "CrossReplicaSum_129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_720/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_720"
  op: "Mul"
  input: "Mul_720/x"
  input: "Square_129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_450"
  op: "AddV2"
  input: "Mul_719"
  input: "Mul_720"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_129"
  op: "Sqrt"
  input: "add_450"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_451/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_451"
  op: "AddV2"
  input: "Sqrt_129"
  input: "add_451/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_132"
  op: "RealDiv"
  input: "add_449"
  input: "add_451"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_566"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_721/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_721"
  op: "Mul"
  input: "mul_721/x"
  input: "ReadVariableOp_566"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_452"
  op: "AddV2"
  input: "truediv_132"
  input: "mul_721"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_722"
  op: "Mul"
  input: "mul_21"
  input: "add_452"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_567"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_133"
  op: "Sub"
  input: "ReadVariableOp_567"
  input: "mul_722"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_387"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_133"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_568"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_387"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_388"
  op: "AssignVariableOp"
  input: "Mul_717/ReadVariableOp/Enter"
  input: "add_449"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_569"
  op: "ReadVariableOp"
  input: "Mul_717/ReadVariableOp/Enter"
  input: "^AssignVariableOp_388"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_389"
  op: "AssignVariableOp"
  input: "Mul_719/ReadVariableOp/Enter"
  input: "add_450"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_570"
  op: "ReadVariableOp"
  input: "Mul_719/ReadVariableOp/Enter"
  input: "^AssignVariableOp_389"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_723/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_723/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_723/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_723/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_723"
  op: "Mul"
  input: "Mul_723/x"
  input: "Mul_723/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_724/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_724"
  op: "Mul"
  input: "Mul_724/x"
  input: "CrossReplicaSum_130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_453"
  op: "AddV2"
  input: "Mul_723"
  input: "Mul_724"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_725/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_725/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_725/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_725/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_725"
  op: "Mul"
  input: "Mul_725/x"
  input: "Mul_725/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_130"
  op: "Square"
  input: "CrossReplicaSum_130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_726/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_726"
  op: "Mul"
  input: "Mul_726/x"
  input: "Square_130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_454"
  op: "AddV2"
  input: "Mul_725"
  input: "Mul_726"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_130"
  op: "Sqrt"
  input: "add_454"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_455/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_455"
  op: "AddV2"
  input: "Sqrt_130"
  input: "add_455/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_133"
  op: "RealDiv"
  input: "add_453"
  input: "add_455"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_727"
  op: "Mul"
  input: "mul_21"
  input: "truediv_133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_571"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_134"
  op: "Sub"
  input: "ReadVariableOp_571"
  input: "mul_727"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_390"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_134"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_572"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_390"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_391"
  op: "AssignVariableOp"
  input: "Mul_723/ReadVariableOp/Enter"
  input: "add_453"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_573"
  op: "ReadVariableOp"
  input: "Mul_723/ReadVariableOp/Enter"
  input: "^AssignVariableOp_391"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_392"
  op: "AssignVariableOp"
  input: "Mul_725/ReadVariableOp/Enter"
  input: "add_454"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_574"
  op: "ReadVariableOp"
  input: "Mul_725/ReadVariableOp/Enter"
  input: "^AssignVariableOp_392"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_728/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_728/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_728/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_728/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_728"
  op: "Mul"
  input: "Mul_728/x"
  input: "Mul_728/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_729/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_729"
  op: "Mul"
  input: "Mul_729/x"
  input: "CrossReplicaSum_131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_456"
  op: "AddV2"
  input: "Mul_728"
  input: "Mul_729"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_730/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_730/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_730/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_730/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_730"
  op: "Mul"
  input: "Mul_730/x"
  input: "Mul_730/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_131"
  op: "Square"
  input: "CrossReplicaSum_131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_731/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_731"
  op: "Mul"
  input: "Mul_731/x"
  input: "Square_131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_457"
  op: "AddV2"
  input: "Mul_730"
  input: "Mul_731"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_131"
  op: "Sqrt"
  input: "add_457"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_458/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_458"
  op: "AddV2"
  input: "Sqrt_131"
  input: "add_458/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_134"
  op: "RealDiv"
  input: "add_456"
  input: "add_458"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_575"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_732/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_732"
  op: "Mul"
  input: "mul_732/x"
  input: "ReadVariableOp_575"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_459"
  op: "AddV2"
  input: "truediv_134"
  input: "mul_732"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_733"
  op: "Mul"
  input: "mul_21"
  input: "add_459"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_576"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_135"
  op: "Sub"
  input: "ReadVariableOp_576"
  input: "mul_733"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_393"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_135"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_577"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_393"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_394"
  op: "AssignVariableOp"
  input: "Mul_728/ReadVariableOp/Enter"
  input: "add_456"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_578"
  op: "ReadVariableOp"
  input: "Mul_728/ReadVariableOp/Enter"
  input: "^AssignVariableOp_394"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_395"
  op: "AssignVariableOp"
  input: "Mul_730/ReadVariableOp/Enter"
  input: "add_457"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_579"
  op: "ReadVariableOp"
  input: "Mul_730/ReadVariableOp/Enter"
  input: "^AssignVariableOp_395"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_m"
  input: "generator/encoder/layer_7/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_v"
  input: "generator/encoder/layer_7/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_734/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_734/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_734/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_734/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_734"
  op: "Mul"
  input: "Mul_734/x"
  input: "Mul_734/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_735/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_735"
  op: "Mul"
  input: "Mul_735/x"
  input: "CrossReplicaSum_132"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_460"
  op: "AddV2"
  input: "Mul_734"
  input: "Mul_735"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_736/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_736/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_736/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_736/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_736"
  op: "Mul"
  input: "Mul_736/x"
  input: "Mul_736/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_132"
  op: "Square"
  input: "CrossReplicaSum_132"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_737/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_737"
  op: "Mul"
  input: "Mul_737/x"
  input: "Square_132"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_461"
  op: "AddV2"
  input: "Mul_736"
  input: "Mul_737"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_132"
  op: "Sqrt"
  input: "add_461"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_462/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_462"
  op: "AddV2"
  input: "Sqrt_132"
  input: "add_462/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_135"
  op: "RealDiv"
  input: "add_460"
  input: "add_462"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_738"
  op: "Mul"
  input: "mul_21"
  input: "truediv_135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_580"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_136"
  op: "Sub"
  input: "ReadVariableOp_580"
  input: "mul_738"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_396"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_136"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_581"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_396"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_397"
  op: "AssignVariableOp"
  input: "Mul_734/ReadVariableOp/Enter"
  input: "add_460"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_582"
  op: "ReadVariableOp"
  input: "Mul_734/ReadVariableOp/Enter"
  input: "^AssignVariableOp_397"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_398"
  op: "AssignVariableOp"
  input: "Mul_736/ReadVariableOp/Enter"
  input: "add_461"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_583"
  op: "ReadVariableOp"
  input: "Mul_736/ReadVariableOp/Enter"
  input: "^AssignVariableOp_398"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_739/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_739/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_739/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_739/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_739"
  op: "Mul"
  input: "Mul_739/x"
  input: "Mul_739/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_740/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_740"
  op: "Mul"
  input: "Mul_740/x"
  input: "CrossReplicaSum_133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_463"
  op: "AddV2"
  input: "Mul_739"
  input: "Mul_740"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_741/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_741/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_741/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_741/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_741"
  op: "Mul"
  input: "Mul_741/x"
  input: "Mul_741/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_133"
  op: "Square"
  input: "CrossReplicaSum_133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_742/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_742"
  op: "Mul"
  input: "Mul_742/x"
  input: "Square_133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_464"
  op: "AddV2"
  input: "Mul_741"
  input: "Mul_742"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_133"
  op: "Sqrt"
  input: "add_464"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_465/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_465"
  op: "AddV2"
  input: "Sqrt_133"
  input: "add_465/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_136"
  op: "RealDiv"
  input: "add_463"
  input: "add_465"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_743"
  op: "Mul"
  input: "mul_21"
  input: "truediv_136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_584"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_137"
  op: "Sub"
  input: "ReadVariableOp_584"
  input: "mul_743"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_399"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_137"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_585"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_399"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_400"
  op: "AssignVariableOp"
  input: "Mul_739/ReadVariableOp/Enter"
  input: "add_463"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_586"
  op: "ReadVariableOp"
  input: "Mul_739/ReadVariableOp/Enter"
  input: "^AssignVariableOp_400"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_401"
  op: "AssignVariableOp"
  input: "Mul_741/ReadVariableOp/Enter"
  input: "add_464"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_587"
  op: "ReadVariableOp"
  input: "Mul_741/ReadVariableOp/Enter"
  input: "^AssignVariableOp_401"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_744/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_744/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_744/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_744/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_744"
  op: "Mul"
  input: "Mul_744/x"
  input: "Mul_744/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_745/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_745"
  op: "Mul"
  input: "Mul_745/x"
  input: "CrossReplicaSum_134"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_466"
  op: "AddV2"
  input: "Mul_744"
  input: "Mul_745"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_746/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_746/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_746/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_746/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_746"
  op: "Mul"
  input: "Mul_746/x"
  input: "Mul_746/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_134"
  op: "Square"
  input: "CrossReplicaSum_134"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_747/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_747"
  op: "Mul"
  input: "Mul_747/x"
  input: "Square_134"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_467"
  op: "AddV2"
  input: "Mul_746"
  input: "Mul_747"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_134"
  op: "Sqrt"
  input: "add_467"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_468/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_468"
  op: "AddV2"
  input: "Sqrt_134"
  input: "add_468/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_137"
  op: "RealDiv"
  input: "add_466"
  input: "add_468"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_748"
  op: "Mul"
  input: "mul_21"
  input: "truediv_137"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_588"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_138"
  op: "Sub"
  input: "ReadVariableOp_588"
  input: "mul_748"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_402"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_138"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_589"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_402"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_403"
  op: "AssignVariableOp"
  input: "Mul_744/ReadVariableOp/Enter"
  input: "add_466"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_590"
  op: "ReadVariableOp"
  input: "Mul_744/ReadVariableOp/Enter"
  input: "^AssignVariableOp_403"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_404"
  op: "AssignVariableOp"
  input: "Mul_746/ReadVariableOp/Enter"
  input: "add_467"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_591"
  op: "ReadVariableOp"
  input: "Mul_746/ReadVariableOp/Enter"
  input: "^AssignVariableOp_404"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_749/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_749/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_749/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_749/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_749"
  op: "Mul"
  input: "Mul_749/x"
  input: "Mul_749/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_750/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_750"
  op: "Mul"
  input: "Mul_750/x"
  input: "CrossReplicaSum_135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_469"
  op: "AddV2"
  input: "Mul_749"
  input: "Mul_750"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_751/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_751/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_751/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_751/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_751"
  op: "Mul"
  input: "Mul_751/x"
  input: "Mul_751/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_135"
  op: "Square"
  input: "CrossReplicaSum_135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_752/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_752"
  op: "Mul"
  input: "Mul_752/x"
  input: "Square_135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_470"
  op: "AddV2"
  input: "Mul_751"
  input: "Mul_752"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_135"
  op: "Sqrt"
  input: "add_470"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_471/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_471"
  op: "AddV2"
  input: "Sqrt_135"
  input: "add_471/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_138"
  op: "RealDiv"
  input: "add_469"
  input: "add_471"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_592"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_753/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_753"
  op: "Mul"
  input: "mul_753/x"
  input: "ReadVariableOp_592"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_472"
  op: "AddV2"
  input: "truediv_138"
  input: "mul_753"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_754"
  op: "Mul"
  input: "mul_21"
  input: "add_472"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_593"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_139"
  op: "Sub"
  input: "ReadVariableOp_593"
  input: "mul_754"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_405"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_139"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_594"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_405"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_406"
  op: "AssignVariableOp"
  input: "Mul_749/ReadVariableOp/Enter"
  input: "add_469"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_595"
  op: "ReadVariableOp"
  input: "Mul_749/ReadVariableOp/Enter"
  input: "^AssignVariableOp_406"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_407"
  op: "AssignVariableOp"
  input: "Mul_751/ReadVariableOp/Enter"
  input: "add_470"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_596"
  op: "ReadVariableOp"
  input: "Mul_751/ReadVariableOp/Enter"
  input: "^AssignVariableOp_407"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_755/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_755/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_755/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_755/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_755"
  op: "Mul"
  input: "Mul_755/x"
  input: "Mul_755/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_756/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_756"
  op: "Mul"
  input: "Mul_756/x"
  input: "CrossReplicaSum_136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_473"
  op: "AddV2"
  input: "Mul_755"
  input: "Mul_756"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_757/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_757/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_757/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_757/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_757"
  op: "Mul"
  input: "Mul_757/x"
  input: "Mul_757/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_136"
  op: "Square"
  input: "CrossReplicaSum_136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_758/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_758"
  op: "Mul"
  input: "Mul_758/x"
  input: "Square_136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_474"
  op: "AddV2"
  input: "Mul_757"
  input: "Mul_758"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_136"
  op: "Sqrt"
  input: "add_474"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_475/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_475"
  op: "AddV2"
  input: "Sqrt_136"
  input: "add_475/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_139"
  op: "RealDiv"
  input: "add_473"
  input: "add_475"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_759"
  op: "Mul"
  input: "mul_21"
  input: "truediv_139"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_597"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_140"
  op: "Sub"
  input: "ReadVariableOp_597"
  input: "mul_759"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_408"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_140"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_598"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_408"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_409"
  op: "AssignVariableOp"
  input: "Mul_755/ReadVariableOp/Enter"
  input: "add_473"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_599"
  op: "ReadVariableOp"
  input: "Mul_755/ReadVariableOp/Enter"
  input: "^AssignVariableOp_409"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_410"
  op: "AssignVariableOp"
  input: "Mul_757/ReadVariableOp/Enter"
  input: "add_474"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_600"
  op: "ReadVariableOp"
  input: "Mul_757/ReadVariableOp/Enter"
  input: "^AssignVariableOp_410"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_760/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_760/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_760/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_760/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_760"
  op: "Mul"
  input: "Mul_760/x"
  input: "Mul_760/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_761/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_761"
  op: "Mul"
  input: "Mul_761/x"
  input: "CrossReplicaSum_137"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_476"
  op: "AddV2"
  input: "Mul_760"
  input: "Mul_761"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_762/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_762/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_762/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_762/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_762"
  op: "Mul"
  input: "Mul_762/x"
  input: "Mul_762/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_137"
  op: "Square"
  input: "CrossReplicaSum_137"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_763/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_763"
  op: "Mul"
  input: "Mul_763/x"
  input: "Square_137"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_477"
  op: "AddV2"
  input: "Mul_762"
  input: "Mul_763"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_137"
  op: "Sqrt"
  input: "add_477"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_478/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_478"
  op: "AddV2"
  input: "Sqrt_137"
  input: "add_478/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_140"
  op: "RealDiv"
  input: "add_476"
  input: "add_478"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_601"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_764/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_764"
  op: "Mul"
  input: "mul_764/x"
  input: "ReadVariableOp_601"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_479"
  op: "AddV2"
  input: "truediv_140"
  input: "mul_764"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_765"
  op: "Mul"
  input: "mul_21"
  input: "add_479"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_602"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_141"
  op: "Sub"
  input: "ReadVariableOp_602"
  input: "mul_765"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_411"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_141"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_603"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_411"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_412"
  op: "AssignVariableOp"
  input: "Mul_760/ReadVariableOp/Enter"
  input: "add_476"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_604"
  op: "ReadVariableOp"
  input: "Mul_760/ReadVariableOp/Enter"
  input: "^AssignVariableOp_412"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_413"
  op: "AssignVariableOp"
  input: "Mul_762/ReadVariableOp/Enter"
  input: "add_477"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_605"
  op: "ReadVariableOp"
  input: "Mul_762/ReadVariableOp/Enter"
  input: "^AssignVariableOp_413"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_766/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_766/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_766/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_766/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_766"
  op: "Mul"
  input: "Mul_766/x"
  input: "Mul_766/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_767/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_767"
  op: "Mul"
  input: "Mul_767/x"
  input: "CrossReplicaSum_138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_480"
  op: "AddV2"
  input: "Mul_766"
  input: "Mul_767"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_768/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_768/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_768/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_768/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_768"
  op: "Mul"
  input: "Mul_768/x"
  input: "Mul_768/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_138"
  op: "Square"
  input: "CrossReplicaSum_138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_769/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_769"
  op: "Mul"
  input: "Mul_769/x"
  input: "Square_138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_481"
  op: "AddV2"
  input: "Mul_768"
  input: "Mul_769"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_138"
  op: "Sqrt"
  input: "add_481"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_482/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_482"
  op: "AddV2"
  input: "Sqrt_138"
  input: "add_482/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_141"
  op: "RealDiv"
  input: "add_480"
  input: "add_482"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_770"
  op: "Mul"
  input: "mul_21"
  input: "truediv_141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_606"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_142"
  op: "Sub"
  input: "ReadVariableOp_606"
  input: "mul_770"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_414"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_142"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_607"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_414"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_415"
  op: "AssignVariableOp"
  input: "Mul_766/ReadVariableOp/Enter"
  input: "add_480"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_608"
  op: "ReadVariableOp"
  input: "Mul_766/ReadVariableOp/Enter"
  input: "^AssignVariableOp_415"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_416"
  op: "AssignVariableOp"
  input: "Mul_768/ReadVariableOp/Enter"
  input: "add_481"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_609"
  op: "ReadVariableOp"
  input: "Mul_768/ReadVariableOp/Enter"
  input: "^AssignVariableOp_416"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_771/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_771/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_771/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_771/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_771"
  op: "Mul"
  input: "Mul_771/x"
  input: "Mul_771/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_772/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_772"
  op: "Mul"
  input: "Mul_772/x"
  input: "CrossReplicaSum_139"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_483"
  op: "AddV2"
  input: "Mul_771"
  input: "Mul_772"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_773/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_773/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_773/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_773/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_773"
  op: "Mul"
  input: "Mul_773/x"
  input: "Mul_773/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_139"
  op: "Square"
  input: "CrossReplicaSum_139"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_774/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_774"
  op: "Mul"
  input: "Mul_774/x"
  input: "Square_139"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_484"
  op: "AddV2"
  input: "Mul_773"
  input: "Mul_774"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_139"
  op: "Sqrt"
  input: "add_484"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_485/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_485"
  op: "AddV2"
  input: "Sqrt_139"
  input: "add_485/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_142"
  op: "RealDiv"
  input: "add_483"
  input: "add_485"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_610"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_775/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_775"
  op: "Mul"
  input: "mul_775/x"
  input: "ReadVariableOp_610"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_486"
  op: "AddV2"
  input: "truediv_142"
  input: "mul_775"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_776"
  op: "Mul"
  input: "mul_21"
  input: "add_486"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_611"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_143"
  op: "Sub"
  input: "ReadVariableOp_611"
  input: "mul_776"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_417"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_143"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_612"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_417"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_418"
  op: "AssignVariableOp"
  input: "Mul_771/ReadVariableOp/Enter"
  input: "add_483"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_613"
  op: "ReadVariableOp"
  input: "Mul_771/ReadVariableOp/Enter"
  input: "^AssignVariableOp_418"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_419"
  op: "AssignVariableOp"
  input: "Mul_773/ReadVariableOp/Enter"
  input: "add_484"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_614"
  op: "ReadVariableOp"
  input: "Mul_773/ReadVariableOp/Enter"
  input: "^AssignVariableOp_419"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_777/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_777/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_777/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_777/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_777"
  op: "Mul"
  input: "Mul_777/x"
  input: "Mul_777/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_778/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_778"
  op: "Mul"
  input: "Mul_778/x"
  input: "CrossReplicaSum_140"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_487"
  op: "AddV2"
  input: "Mul_777"
  input: "Mul_778"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_779/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_779/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_779/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_779/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_779"
  op: "Mul"
  input: "Mul_779/x"
  input: "Mul_779/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_140"
  op: "Square"
  input: "CrossReplicaSum_140"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_780/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_780"
  op: "Mul"
  input: "Mul_780/x"
  input: "Square_140"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_488"
  op: "AddV2"
  input: "Mul_779"
  input: "Mul_780"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_140"
  op: "Sqrt"
  input: "add_488"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_489/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_489"
  op: "AddV2"
  input: "Sqrt_140"
  input: "add_489/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_143"
  op: "RealDiv"
  input: "add_487"
  input: "add_489"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_781"
  op: "Mul"
  input: "mul_21"
  input: "truediv_143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_615"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_144"
  op: "Sub"
  input: "ReadVariableOp_615"
  input: "mul_781"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_420"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_144"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_616"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_420"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_421"
  op: "AssignVariableOp"
  input: "Mul_777/ReadVariableOp/Enter"
  input: "add_487"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_617"
  op: "ReadVariableOp"
  input: "Mul_777/ReadVariableOp/Enter"
  input: "^AssignVariableOp_421"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_422"
  op: "AssignVariableOp"
  input: "Mul_779/ReadVariableOp/Enter"
  input: "add_488"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_618"
  op: "ReadVariableOp"
  input: "Mul_779/ReadVariableOp/Enter"
  input: "^AssignVariableOp_422"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_782/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_782/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_782/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_782/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_782"
  op: "Mul"
  input: "Mul_782/x"
  input: "Mul_782/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_783/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_783"
  op: "Mul"
  input: "Mul_783/x"
  input: "CrossReplicaSum_141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_490"
  op: "AddV2"
  input: "Mul_782"
  input: "Mul_783"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_784/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_784/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_784/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_784/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_784"
  op: "Mul"
  input: "Mul_784/x"
  input: "Mul_784/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_141"
  op: "Square"
  input: "CrossReplicaSum_141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_785/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_785"
  op: "Mul"
  input: "Mul_785/x"
  input: "Square_141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_491"
  op: "AddV2"
  input: "Mul_784"
  input: "Mul_785"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_141"
  op: "Sqrt"
  input: "add_491"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_492/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_492"
  op: "AddV2"
  input: "Sqrt_141"
  input: "add_492/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_144"
  op: "RealDiv"
  input: "add_490"
  input: "add_492"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_619"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_786/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_786"
  op: "Mul"
  input: "mul_786/x"
  input: "ReadVariableOp_619"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_493"
  op: "AddV2"
  input: "truediv_144"
  input: "mul_786"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_787"
  op: "Mul"
  input: "mul_21"
  input: "add_493"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_620"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_145"
  op: "Sub"
  input: "ReadVariableOp_620"
  input: "mul_787"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_423"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_145"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_621"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_423"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_424"
  op: "AssignVariableOp"
  input: "Mul_782/ReadVariableOp/Enter"
  input: "add_490"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_622"
  op: "ReadVariableOp"
  input: "Mul_782/ReadVariableOp/Enter"
  input: "^AssignVariableOp_424"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_425"
  op: "AssignVariableOp"
  input: "Mul_784/ReadVariableOp/Enter"
  input: "add_491"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_623"
  op: "ReadVariableOp"
  input: "Mul_784/ReadVariableOp/Enter"
  input: "^AssignVariableOp_425"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_788/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_788/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_788/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_788/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_788"
  op: "Mul"
  input: "Mul_788/x"
  input: "Mul_788/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_789/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_789"
  op: "Mul"
  input: "Mul_789/x"
  input: "CrossReplicaSum_142"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_494"
  op: "AddV2"
  input: "Mul_788"
  input: "Mul_789"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_790/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_790/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_790/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_790/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_790"
  op: "Mul"
  input: "Mul_790/x"
  input: "Mul_790/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_142"
  op: "Square"
  input: "CrossReplicaSum_142"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_791/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_791"
  op: "Mul"
  input: "Mul_791/x"
  input: "Square_142"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_495"
  op: "AddV2"
  input: "Mul_790"
  input: "Mul_791"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_142"
  op: "Sqrt"
  input: "add_495"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_496/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_496"
  op: "AddV2"
  input: "Sqrt_142"
  input: "add_496/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_145"
  op: "RealDiv"
  input: "add_494"
  input: "add_496"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_792"
  op: "Mul"
  input: "mul_21"
  input: "truediv_145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_624"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_146"
  op: "Sub"
  input: "ReadVariableOp_624"
  input: "mul_792"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_426"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_146"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_625"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_426"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_427"
  op: "AssignVariableOp"
  input: "Mul_788/ReadVariableOp/Enter"
  input: "add_494"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_626"
  op: "ReadVariableOp"
  input: "Mul_788/ReadVariableOp/Enter"
  input: "^AssignVariableOp_427"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_428"
  op: "AssignVariableOp"
  input: "Mul_790/ReadVariableOp/Enter"
  input: "add_495"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_627"
  op: "ReadVariableOp"
  input: "Mul_790/ReadVariableOp/Enter"
  input: "^AssignVariableOp_428"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_793/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_793/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_793/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_793/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_793"
  op: "Mul"
  input: "Mul_793/x"
  input: "Mul_793/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_794/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_794"
  op: "Mul"
  input: "Mul_794/x"
  input: "CrossReplicaSum_143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_497"
  op: "AddV2"
  input: "Mul_793"
  input: "Mul_794"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_795/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_795/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_795/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_795/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_795"
  op: "Mul"
  input: "Mul_795/x"
  input: "Mul_795/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_143"
  op: "Square"
  input: "CrossReplicaSum_143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_796/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_796"
  op: "Mul"
  input: "Mul_796/x"
  input: "Square_143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_498"
  op: "AddV2"
  input: "Mul_795"
  input: "Mul_796"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_143"
  op: "Sqrt"
  input: "add_498"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_499/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_499"
  op: "AddV2"
  input: "Sqrt_143"
  input: "add_499/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_146"
  op: "RealDiv"
  input: "add_497"
  input: "add_499"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_797"
  op: "Mul"
  input: "mul_21"
  input: "truediv_146"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_628"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_147"
  op: "Sub"
  input: "ReadVariableOp_628"
  input: "mul_797"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_429"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_147"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_629"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_429"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_430"
  op: "AssignVariableOp"
  input: "Mul_793/ReadVariableOp/Enter"
  input: "add_497"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_630"
  op: "ReadVariableOp"
  input: "Mul_793/ReadVariableOp/Enter"
  input: "^AssignVariableOp_430"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_431"
  op: "AssignVariableOp"
  input: "Mul_795/ReadVariableOp/Enter"
  input: "add_498"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_631"
  op: "ReadVariableOp"
  input: "Mul_795/ReadVariableOp/Enter"
  input: "^AssignVariableOp_431"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_798/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_798/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_798/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_798/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_798"
  op: "Mul"
  input: "Mul_798/x"
  input: "Mul_798/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_799/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_799"
  op: "Mul"
  input: "Mul_799/x"
  input: "CrossReplicaSum_144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_500"
  op: "AddV2"
  input: "Mul_798"
  input: "Mul_799"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_800/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_800/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_800/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_800/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_800"
  op: "Mul"
  input: "Mul_800/x"
  input: "Mul_800/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_144"
  op: "Square"
  input: "CrossReplicaSum_144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_801/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_801"
  op: "Mul"
  input: "Mul_801/x"
  input: "Square_144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_501"
  op: "AddV2"
  input: "Mul_800"
  input: "Mul_801"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_144"
  op: "Sqrt"
  input: "add_501"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_502/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_502"
  op: "AddV2"
  input: "Sqrt_144"
  input: "add_502/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_147"
  op: "RealDiv"
  input: "add_500"
  input: "add_502"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_802"
  op: "Mul"
  input: "mul_21"
  input: "truediv_147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_632"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_148"
  op: "Sub"
  input: "ReadVariableOp_632"
  input: "mul_802"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_432"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_148"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_633"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_432"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_433"
  op: "AssignVariableOp"
  input: "Mul_798/ReadVariableOp/Enter"
  input: "add_500"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_634"
  op: "ReadVariableOp"
  input: "Mul_798/ReadVariableOp/Enter"
  input: "^AssignVariableOp_433"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_434"
  op: "AssignVariableOp"
  input: "Mul_800/ReadVariableOp/Enter"
  input: "add_501"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_635"
  op: "ReadVariableOp"
  input: "Mul_800/ReadVariableOp/Enter"
  input: "^AssignVariableOp_434"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_803/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_803/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_803/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_803/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_803"
  op: "Mul"
  input: "Mul_803/x"
  input: "Mul_803/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_804/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_804"
  op: "Mul"
  input: "Mul_804/x"
  input: "CrossReplicaSum_145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_503"
  op: "AddV2"
  input: "Mul_803"
  input: "Mul_804"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_805/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_805/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_805/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_805/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_805"
  op: "Mul"
  input: "Mul_805/x"
  input: "Mul_805/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_145"
  op: "Square"
  input: "CrossReplicaSum_145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_806/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_806"
  op: "Mul"
  input: "Mul_806/x"
  input: "Square_145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_504"
  op: "AddV2"
  input: "Mul_805"
  input: "Mul_806"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_145"
  op: "Sqrt"
  input: "add_504"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_505/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_505"
  op: "AddV2"
  input: "Sqrt_145"
  input: "add_505/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_148"
  op: "RealDiv"
  input: "add_503"
  input: "add_505"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_636"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_807/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_807"
  op: "Mul"
  input: "mul_807/x"
  input: "ReadVariableOp_636"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_506"
  op: "AddV2"
  input: "truediv_148"
  input: "mul_807"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_808"
  op: "Mul"
  input: "mul_21"
  input: "add_506"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_637"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_149"
  op: "Sub"
  input: "ReadVariableOp_637"
  input: "mul_808"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_435"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_149"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_638"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_435"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_436"
  op: "AssignVariableOp"
  input: "Mul_803/ReadVariableOp/Enter"
  input: "add_503"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_639"
  op: "ReadVariableOp"
  input: "Mul_803/ReadVariableOp/Enter"
  input: "^AssignVariableOp_436"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_437"
  op: "AssignVariableOp"
  input: "Mul_805/ReadVariableOp/Enter"
  input: "add_504"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_640"
  op: "ReadVariableOp"
  input: "Mul_805/ReadVariableOp/Enter"
  input: "^AssignVariableOp_437"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_809/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_809/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_809/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_809/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_809"
  op: "Mul"
  input: "Mul_809/x"
  input: "Mul_809/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_810/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_810"
  op: "Mul"
  input: "Mul_810/x"
  input: "CrossReplicaSum_146"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_507"
  op: "AddV2"
  input: "Mul_809"
  input: "Mul_810"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_811/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_811/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_811/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_811/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_811"
  op: "Mul"
  input: "Mul_811/x"
  input: "Mul_811/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_146"
  op: "Square"
  input: "CrossReplicaSum_146"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_812/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_812"
  op: "Mul"
  input: "Mul_812/x"
  input: "Square_146"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_508"
  op: "AddV2"
  input: "Mul_811"
  input: "Mul_812"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_146"
  op: "Sqrt"
  input: "add_508"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_509/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_509"
  op: "AddV2"
  input: "Sqrt_146"
  input: "add_509/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_149"
  op: "RealDiv"
  input: "add_507"
  input: "add_509"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_813"
  op: "Mul"
  input: "mul_21"
  input: "truediv_149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_641"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_150"
  op: "Sub"
  input: "ReadVariableOp_641"
  input: "mul_813"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_438"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_150"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_642"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_438"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_439"
  op: "AssignVariableOp"
  input: "Mul_809/ReadVariableOp/Enter"
  input: "add_507"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_643"
  op: "ReadVariableOp"
  input: "Mul_809/ReadVariableOp/Enter"
  input: "^AssignVariableOp_439"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_440"
  op: "AssignVariableOp"
  input: "Mul_811/ReadVariableOp/Enter"
  input: "add_508"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_644"
  op: "ReadVariableOp"
  input: "Mul_811/ReadVariableOp/Enter"
  input: "^AssignVariableOp_440"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_814/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_814/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_814/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_814/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_814"
  op: "Mul"
  input: "Mul_814/x"
  input: "Mul_814/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_815/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_815"
  op: "Mul"
  input: "Mul_815/x"
  input: "CrossReplicaSum_147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_510"
  op: "AddV2"
  input: "Mul_814"
  input: "Mul_815"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_816/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_816/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_816/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_816/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_816"
  op: "Mul"
  input: "Mul_816/x"
  input: "Mul_816/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_147"
  op: "Square"
  input: "CrossReplicaSum_147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_817/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_817"
  op: "Mul"
  input: "Mul_817/x"
  input: "Square_147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_511"
  op: "AddV2"
  input: "Mul_816"
  input: "Mul_817"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_147"
  op: "Sqrt"
  input: "add_511"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_512/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_512"
  op: "AddV2"
  input: "Sqrt_147"
  input: "add_512/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_150"
  op: "RealDiv"
  input: "add_510"
  input: "add_512"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_645"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_818/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_818"
  op: "Mul"
  input: "mul_818/x"
  input: "ReadVariableOp_645"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_513"
  op: "AddV2"
  input: "truediv_150"
  input: "mul_818"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_819"
  op: "Mul"
  input: "mul_21"
  input: "add_513"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_646"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_151"
  op: "Sub"
  input: "ReadVariableOp_646"
  input: "mul_819"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_441"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_151"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_647"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_441"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_442"
  op: "AssignVariableOp"
  input: "Mul_814/ReadVariableOp/Enter"
  input: "add_510"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_648"
  op: "ReadVariableOp"
  input: "Mul_814/ReadVariableOp/Enter"
  input: "^AssignVariableOp_442"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_443"
  op: "AssignVariableOp"
  input: "Mul_816/ReadVariableOp/Enter"
  input: "add_511"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_649"
  op: "ReadVariableOp"
  input: "Mul_816/ReadVariableOp/Enter"
  input: "^AssignVariableOp_443"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_m"
  input: "generator/encoder/layer_8/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_v"
  input: "generator/encoder/layer_8/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_820/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_820/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_820/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_820/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_820"
  op: "Mul"
  input: "Mul_820/x"
  input: "Mul_820/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_821/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_821"
  op: "Mul"
  input: "Mul_821/x"
  input: "CrossReplicaSum_148"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_514"
  op: "AddV2"
  input: "Mul_820"
  input: "Mul_821"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_822/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_822/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_822/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_822/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_822"
  op: "Mul"
  input: "Mul_822/x"
  input: "Mul_822/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_148"
  op: "Square"
  input: "CrossReplicaSum_148"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_823/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_823"
  op: "Mul"
  input: "Mul_823/x"
  input: "Square_148"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_515"
  op: "AddV2"
  input: "Mul_822"
  input: "Mul_823"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_148"
  op: "Sqrt"
  input: "add_515"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_516/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_516"
  op: "AddV2"
  input: "Sqrt_148"
  input: "add_516/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_151"
  op: "RealDiv"
  input: "add_514"
  input: "add_516"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_824"
  op: "Mul"
  input: "mul_21"
  input: "truediv_151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_650"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_152"
  op: "Sub"
  input: "ReadVariableOp_650"
  input: "mul_824"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_444"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_152"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_651"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_444"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_445"
  op: "AssignVariableOp"
  input: "Mul_820/ReadVariableOp/Enter"
  input: "add_514"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_652"
  op: "ReadVariableOp"
  input: "Mul_820/ReadVariableOp/Enter"
  input: "^AssignVariableOp_445"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_446"
  op: "AssignVariableOp"
  input: "Mul_822/ReadVariableOp/Enter"
  input: "add_515"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_653"
  op: "ReadVariableOp"
  input: "Mul_822/ReadVariableOp/Enter"
  input: "^AssignVariableOp_446"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_825/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_825/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_825/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_825/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_825"
  op: "Mul"
  input: "Mul_825/x"
  input: "Mul_825/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_826/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_826"
  op: "Mul"
  input: "Mul_826/x"
  input: "CrossReplicaSum_149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_517"
  op: "AddV2"
  input: "Mul_825"
  input: "Mul_826"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_827/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_827/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_827/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_827/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_827"
  op: "Mul"
  input: "Mul_827/x"
  input: "Mul_827/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_149"
  op: "Square"
  input: "CrossReplicaSum_149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_828/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_828"
  op: "Mul"
  input: "Mul_828/x"
  input: "Square_149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_518"
  op: "AddV2"
  input: "Mul_827"
  input: "Mul_828"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_149"
  op: "Sqrt"
  input: "add_518"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_519/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_519"
  op: "AddV2"
  input: "Sqrt_149"
  input: "add_519/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_152"
  op: "RealDiv"
  input: "add_517"
  input: "add_519"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_829"
  op: "Mul"
  input: "mul_21"
  input: "truediv_152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_654"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_153"
  op: "Sub"
  input: "ReadVariableOp_654"
  input: "mul_829"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_447"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_153"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_655"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_447"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_448"
  op: "AssignVariableOp"
  input: "Mul_825/ReadVariableOp/Enter"
  input: "add_517"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_656"
  op: "ReadVariableOp"
  input: "Mul_825/ReadVariableOp/Enter"
  input: "^AssignVariableOp_448"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_449"
  op: "AssignVariableOp"
  input: "Mul_827/ReadVariableOp/Enter"
  input: "add_518"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_657"
  op: "ReadVariableOp"
  input: "Mul_827/ReadVariableOp/Enter"
  input: "^AssignVariableOp_449"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_830/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_830/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_830/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_830/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_830"
  op: "Mul"
  input: "Mul_830/x"
  input: "Mul_830/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_831/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_831"
  op: "Mul"
  input: "Mul_831/x"
  input: "CrossReplicaSum_150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_520"
  op: "AddV2"
  input: "Mul_830"
  input: "Mul_831"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_832/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_832/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_832/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_832/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_832"
  op: "Mul"
  input: "Mul_832/x"
  input: "Mul_832/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_150"
  op: "Square"
  input: "CrossReplicaSum_150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_833/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_833"
  op: "Mul"
  input: "Mul_833/x"
  input: "Square_150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_521"
  op: "AddV2"
  input: "Mul_832"
  input: "Mul_833"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_150"
  op: "Sqrt"
  input: "add_521"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_522/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_522"
  op: "AddV2"
  input: "Sqrt_150"
  input: "add_522/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_153"
  op: "RealDiv"
  input: "add_520"
  input: "add_522"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_834"
  op: "Mul"
  input: "mul_21"
  input: "truediv_153"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_658"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_154"
  op: "Sub"
  input: "ReadVariableOp_658"
  input: "mul_834"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_450"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_154"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_659"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_450"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_451"
  op: "AssignVariableOp"
  input: "Mul_830/ReadVariableOp/Enter"
  input: "add_520"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_660"
  op: "ReadVariableOp"
  input: "Mul_830/ReadVariableOp/Enter"
  input: "^AssignVariableOp_451"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_452"
  op: "AssignVariableOp"
  input: "Mul_832/ReadVariableOp/Enter"
  input: "add_521"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_661"
  op: "ReadVariableOp"
  input: "Mul_832/ReadVariableOp/Enter"
  input: "^AssignVariableOp_452"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_835/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_835/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_835/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_835/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_835"
  op: "Mul"
  input: "Mul_835/x"
  input: "Mul_835/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_836/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_836"
  op: "Mul"
  input: "Mul_836/x"
  input: "CrossReplicaSum_151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_523"
  op: "AddV2"
  input: "Mul_835"
  input: "Mul_836"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_837/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_837/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_837/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_837/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_837"
  op: "Mul"
  input: "Mul_837/x"
  input: "Mul_837/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_151"
  op: "Square"
  input: "CrossReplicaSum_151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_838/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_838"
  op: "Mul"
  input: "Mul_838/x"
  input: "Square_151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_524"
  op: "AddV2"
  input: "Mul_837"
  input: "Mul_838"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_151"
  op: "Sqrt"
  input: "add_524"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_525/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_525"
  op: "AddV2"
  input: "Sqrt_151"
  input: "add_525/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_154"
  op: "RealDiv"
  input: "add_523"
  input: "add_525"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_662"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_839/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_839"
  op: "Mul"
  input: "mul_839/x"
  input: "ReadVariableOp_662"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_526"
  op: "AddV2"
  input: "truediv_154"
  input: "mul_839"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_840"
  op: "Mul"
  input: "mul_21"
  input: "add_526"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_663"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_155"
  op: "Sub"
  input: "ReadVariableOp_663"
  input: "mul_840"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_453"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_155"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_664"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_453"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_454"
  op: "AssignVariableOp"
  input: "Mul_835/ReadVariableOp/Enter"
  input: "add_523"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_665"
  op: "ReadVariableOp"
  input: "Mul_835/ReadVariableOp/Enter"
  input: "^AssignVariableOp_454"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_455"
  op: "AssignVariableOp"
  input: "Mul_837/ReadVariableOp/Enter"
  input: "add_524"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_666"
  op: "ReadVariableOp"
  input: "Mul_837/ReadVariableOp/Enter"
  input: "^AssignVariableOp_455"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_841/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_841/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_841/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_841/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_841"
  op: "Mul"
  input: "Mul_841/x"
  input: "Mul_841/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_842/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_842"
  op: "Mul"
  input: "Mul_842/x"
  input: "CrossReplicaSum_152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_527"
  op: "AddV2"
  input: "Mul_841"
  input: "Mul_842"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_843/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_843/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_843/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_843/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_843"
  op: "Mul"
  input: "Mul_843/x"
  input: "Mul_843/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_152"
  op: "Square"
  input: "CrossReplicaSum_152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_844/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_844"
  op: "Mul"
  input: "Mul_844/x"
  input: "Square_152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_528"
  op: "AddV2"
  input: "Mul_843"
  input: "Mul_844"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_152"
  op: "Sqrt"
  input: "add_528"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_529/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_529"
  op: "AddV2"
  input: "Sqrt_152"
  input: "add_529/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_155"
  op: "RealDiv"
  input: "add_527"
  input: "add_529"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_845"
  op: "Mul"
  input: "mul_21"
  input: "truediv_155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_667"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_156"
  op: "Sub"
  input: "ReadVariableOp_667"
  input: "mul_845"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_456"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_156"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_668"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_456"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_457"
  op: "AssignVariableOp"
  input: "Mul_841/ReadVariableOp/Enter"
  input: "add_527"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_669"
  op: "ReadVariableOp"
  input: "Mul_841/ReadVariableOp/Enter"
  input: "^AssignVariableOp_457"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_458"
  op: "AssignVariableOp"
  input: "Mul_843/ReadVariableOp/Enter"
  input: "add_528"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_670"
  op: "ReadVariableOp"
  input: "Mul_843/ReadVariableOp/Enter"
  input: "^AssignVariableOp_458"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_846/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_846/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_846/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_846/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_846"
  op: "Mul"
  input: "Mul_846/x"
  input: "Mul_846/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_847/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_847"
  op: "Mul"
  input: "Mul_847/x"
  input: "CrossReplicaSum_153"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_530"
  op: "AddV2"
  input: "Mul_846"
  input: "Mul_847"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_848/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_848/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_848/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_848/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_848"
  op: "Mul"
  input: "Mul_848/x"
  input: "Mul_848/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_153"
  op: "Square"
  input: "CrossReplicaSum_153"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_849/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_849"
  op: "Mul"
  input: "Mul_849/x"
  input: "Square_153"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_531"
  op: "AddV2"
  input: "Mul_848"
  input: "Mul_849"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_153"
  op: "Sqrt"
  input: "add_531"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_532/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_532"
  op: "AddV2"
  input: "Sqrt_153"
  input: "add_532/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_156"
  op: "RealDiv"
  input: "add_530"
  input: "add_532"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_671"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_850/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_850"
  op: "Mul"
  input: "mul_850/x"
  input: "ReadVariableOp_671"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_533"
  op: "AddV2"
  input: "truediv_156"
  input: "mul_850"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_851"
  op: "Mul"
  input: "mul_21"
  input: "add_533"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_672"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_157"
  op: "Sub"
  input: "ReadVariableOp_672"
  input: "mul_851"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_459"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_157"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_673"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_459"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_460"
  op: "AssignVariableOp"
  input: "Mul_846/ReadVariableOp/Enter"
  input: "add_530"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_674"
  op: "ReadVariableOp"
  input: "Mul_846/ReadVariableOp/Enter"
  input: "^AssignVariableOp_460"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_461"
  op: "AssignVariableOp"
  input: "Mul_848/ReadVariableOp/Enter"
  input: "add_531"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_675"
  op: "ReadVariableOp"
  input: "Mul_848/ReadVariableOp/Enter"
  input: "^AssignVariableOp_461"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_852/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_852/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_852/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_852/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_852"
  op: "Mul"
  input: "Mul_852/x"
  input: "Mul_852/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_853/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_853"
  op: "Mul"
  input: "Mul_853/x"
  input: "CrossReplicaSum_154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_534"
  op: "AddV2"
  input: "Mul_852"
  input: "Mul_853"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_854/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_854/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_854/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_854/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_854"
  op: "Mul"
  input: "Mul_854/x"
  input: "Mul_854/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_154"
  op: "Square"
  input: "CrossReplicaSum_154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_855/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_855"
  op: "Mul"
  input: "Mul_855/x"
  input: "Square_154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_535"
  op: "AddV2"
  input: "Mul_854"
  input: "Mul_855"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_154"
  op: "Sqrt"
  input: "add_535"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_536/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_536"
  op: "AddV2"
  input: "Sqrt_154"
  input: "add_536/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_157"
  op: "RealDiv"
  input: "add_534"
  input: "add_536"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_856"
  op: "Mul"
  input: "mul_21"
  input: "truediv_157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_676"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_158"
  op: "Sub"
  input: "ReadVariableOp_676"
  input: "mul_856"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_462"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_158"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_677"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_462"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_463"
  op: "AssignVariableOp"
  input: "Mul_852/ReadVariableOp/Enter"
  input: "add_534"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_678"
  op: "ReadVariableOp"
  input: "Mul_852/ReadVariableOp/Enter"
  input: "^AssignVariableOp_463"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_464"
  op: "AssignVariableOp"
  input: "Mul_854/ReadVariableOp/Enter"
  input: "add_535"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_679"
  op: "ReadVariableOp"
  input: "Mul_854/ReadVariableOp/Enter"
  input: "^AssignVariableOp_464"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_857/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_857/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_857/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_857/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_857"
  op: "Mul"
  input: "Mul_857/x"
  input: "Mul_857/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_858/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_858"
  op: "Mul"
  input: "Mul_858/x"
  input: "CrossReplicaSum_155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_537"
  op: "AddV2"
  input: "Mul_857"
  input: "Mul_858"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_859/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_859/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_859/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_859/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_859"
  op: "Mul"
  input: "Mul_859/x"
  input: "Mul_859/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_155"
  op: "Square"
  input: "CrossReplicaSum_155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_860/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_860"
  op: "Mul"
  input: "Mul_860/x"
  input: "Square_155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_538"
  op: "AddV2"
  input: "Mul_859"
  input: "Mul_860"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_155"
  op: "Sqrt"
  input: "add_538"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_539/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_539"
  op: "AddV2"
  input: "Sqrt_155"
  input: "add_539/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_158"
  op: "RealDiv"
  input: "add_537"
  input: "add_539"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_680"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_861/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_861"
  op: "Mul"
  input: "mul_861/x"
  input: "ReadVariableOp_680"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_540"
  op: "AddV2"
  input: "truediv_158"
  input: "mul_861"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_862"
  op: "Mul"
  input: "mul_21"
  input: "add_540"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_681"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_159"
  op: "Sub"
  input: "ReadVariableOp_681"
  input: "mul_862"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_465"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_159"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_682"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_465"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_466"
  op: "AssignVariableOp"
  input: "Mul_857/ReadVariableOp/Enter"
  input: "add_537"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_683"
  op: "ReadVariableOp"
  input: "Mul_857/ReadVariableOp/Enter"
  input: "^AssignVariableOp_466"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_467"
  op: "AssignVariableOp"
  input: "Mul_859/ReadVariableOp/Enter"
  input: "add_538"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_684"
  op: "ReadVariableOp"
  input: "Mul_859/ReadVariableOp/Enter"
  input: "^AssignVariableOp_467"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_863/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_863/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_863/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_863/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_863"
  op: "Mul"
  input: "Mul_863/x"
  input: "Mul_863/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_864/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_864"
  op: "Mul"
  input: "Mul_864/x"
  input: "CrossReplicaSum_156"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_541"
  op: "AddV2"
  input: "Mul_863"
  input: "Mul_864"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_865/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_865/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_865/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_865/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_865"
  op: "Mul"
  input: "Mul_865/x"
  input: "Mul_865/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_156"
  op: "Square"
  input: "CrossReplicaSum_156"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_866/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_866"
  op: "Mul"
  input: "Mul_866/x"
  input: "Square_156"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_542"
  op: "AddV2"
  input: "Mul_865"
  input: "Mul_866"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_156"
  op: "Sqrt"
  input: "add_542"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_543/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_543"
  op: "AddV2"
  input: "Sqrt_156"
  input: "add_543/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_159"
  op: "RealDiv"
  input: "add_541"
  input: "add_543"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_867"
  op: "Mul"
  input: "mul_21"
  input: "truediv_159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_685"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_160"
  op: "Sub"
  input: "ReadVariableOp_685"
  input: "mul_867"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_468"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_160"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_686"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_468"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_469"
  op: "AssignVariableOp"
  input: "Mul_863/ReadVariableOp/Enter"
  input: "add_541"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_687"
  op: "ReadVariableOp"
  input: "Mul_863/ReadVariableOp/Enter"
  input: "^AssignVariableOp_469"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_470"
  op: "AssignVariableOp"
  input: "Mul_865/ReadVariableOp/Enter"
  input: "add_542"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_688"
  op: "ReadVariableOp"
  input: "Mul_865/ReadVariableOp/Enter"
  input: "^AssignVariableOp_470"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_868/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_868/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_868/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_868/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_868"
  op: "Mul"
  input: "Mul_868/x"
  input: "Mul_868/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_869/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_869"
  op: "Mul"
  input: "Mul_869/x"
  input: "CrossReplicaSum_157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_544"
  op: "AddV2"
  input: "Mul_868"
  input: "Mul_869"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_870/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_870/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_870/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_870/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_870"
  op: "Mul"
  input: "Mul_870/x"
  input: "Mul_870/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_157"
  op: "Square"
  input: "CrossReplicaSum_157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_871/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_871"
  op: "Mul"
  input: "Mul_871/x"
  input: "Square_157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_545"
  op: "AddV2"
  input: "Mul_870"
  input: "Mul_871"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_157"
  op: "Sqrt"
  input: "add_545"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_546/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_546"
  op: "AddV2"
  input: "Sqrt_157"
  input: "add_546/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_160"
  op: "RealDiv"
  input: "add_544"
  input: "add_546"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_689"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_872/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_872"
  op: "Mul"
  input: "mul_872/x"
  input: "ReadVariableOp_689"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_547"
  op: "AddV2"
  input: "truediv_160"
  input: "mul_872"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_873"
  op: "Mul"
  input: "mul_21"
  input: "add_547"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_690"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_161"
  op: "Sub"
  input: "ReadVariableOp_690"
  input: "mul_873"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_471"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_161"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_691"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_471"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_472"
  op: "AssignVariableOp"
  input: "Mul_868/ReadVariableOp/Enter"
  input: "add_544"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_692"
  op: "ReadVariableOp"
  input: "Mul_868/ReadVariableOp/Enter"
  input: "^AssignVariableOp_472"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_473"
  op: "AssignVariableOp"
  input: "Mul_870/ReadVariableOp/Enter"
  input: "add_545"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_693"
  op: "ReadVariableOp"
  input: "Mul_870/ReadVariableOp/Enter"
  input: "^AssignVariableOp_473"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_874/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_874/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_874/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_874/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_874"
  op: "Mul"
  input: "Mul_874/x"
  input: "Mul_874/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_875/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_875"
  op: "Mul"
  input: "Mul_875/x"
  input: "CrossReplicaSum_158"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_548"
  op: "AddV2"
  input: "Mul_874"
  input: "Mul_875"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_876/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_876/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_876/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_876/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_876"
  op: "Mul"
  input: "Mul_876/x"
  input: "Mul_876/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_158"
  op: "Square"
  input: "CrossReplicaSum_158"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_877/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_877"
  op: "Mul"
  input: "Mul_877/x"
  input: "Square_158"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_549"
  op: "AddV2"
  input: "Mul_876"
  input: "Mul_877"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_158"
  op: "Sqrt"
  input: "add_549"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_550/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_550"
  op: "AddV2"
  input: "Sqrt_158"
  input: "add_550/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_161"
  op: "RealDiv"
  input: "add_548"
  input: "add_550"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_878"
  op: "Mul"
  input: "mul_21"
  input: "truediv_161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_694"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_162"
  op: "Sub"
  input: "ReadVariableOp_694"
  input: "mul_878"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_474"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_162"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_695"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_474"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_475"
  op: "AssignVariableOp"
  input: "Mul_874/ReadVariableOp/Enter"
  input: "add_548"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_696"
  op: "ReadVariableOp"
  input: "Mul_874/ReadVariableOp/Enter"
  input: "^AssignVariableOp_475"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_476"
  op: "AssignVariableOp"
  input: "Mul_876/ReadVariableOp/Enter"
  input: "add_549"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_697"
  op: "ReadVariableOp"
  input: "Mul_876/ReadVariableOp/Enter"
  input: "^AssignVariableOp_476"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_879/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_879/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_879/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_879/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_879"
  op: "Mul"
  input: "Mul_879/x"
  input: "Mul_879/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_880/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_880"
  op: "Mul"
  input: "Mul_880/x"
  input: "CrossReplicaSum_159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_551"
  op: "AddV2"
  input: "Mul_879"
  input: "Mul_880"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_881/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_881/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_881/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_881/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_881"
  op: "Mul"
  input: "Mul_881/x"
  input: "Mul_881/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_159"
  op: "Square"
  input: "CrossReplicaSum_159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_882/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_882"
  op: "Mul"
  input: "Mul_882/x"
  input: "Square_159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_552"
  op: "AddV2"
  input: "Mul_881"
  input: "Mul_882"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_159"
  op: "Sqrt"
  input: "add_552"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_553/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_553"
  op: "AddV2"
  input: "Sqrt_159"
  input: "add_553/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_162"
  op: "RealDiv"
  input: "add_551"
  input: "add_553"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_883"
  op: "Mul"
  input: "mul_21"
  input: "truediv_162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_698"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_163"
  op: "Sub"
  input: "ReadVariableOp_698"
  input: "mul_883"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_477"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_163"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_699"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_477"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_478"
  op: "AssignVariableOp"
  input: "Mul_879/ReadVariableOp/Enter"
  input: "add_551"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_700"
  op: "ReadVariableOp"
  input: "Mul_879/ReadVariableOp/Enter"
  input: "^AssignVariableOp_478"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_479"
  op: "AssignVariableOp"
  input: "Mul_881/ReadVariableOp/Enter"
  input: "add_552"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_701"
  op: "ReadVariableOp"
  input: "Mul_881/ReadVariableOp/Enter"
  input: "^AssignVariableOp_479"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_884/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_884/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_884/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_884/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_884"
  op: "Mul"
  input: "Mul_884/x"
  input: "Mul_884/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_885/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_885"
  op: "Mul"
  input: "Mul_885/x"
  input: "CrossReplicaSum_160"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_554"
  op: "AddV2"
  input: "Mul_884"
  input: "Mul_885"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_886/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_886/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_886/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_886/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_886"
  op: "Mul"
  input: "Mul_886/x"
  input: "Mul_886/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_160"
  op: "Square"
  input: "CrossReplicaSum_160"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_887/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_887"
  op: "Mul"
  input: "Mul_887/x"
  input: "Square_160"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_555"
  op: "AddV2"
  input: "Mul_886"
  input: "Mul_887"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_160"
  op: "Sqrt"
  input: "add_555"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_556/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_556"
  op: "AddV2"
  input: "Sqrt_160"
  input: "add_556/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_163"
  op: "RealDiv"
  input: "add_554"
  input: "add_556"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_888"
  op: "Mul"
  input: "mul_21"
  input: "truediv_163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_702"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_164"
  op: "Sub"
  input: "ReadVariableOp_702"
  input: "mul_888"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_480"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_164"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_703"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_480"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_481"
  op: "AssignVariableOp"
  input: "Mul_884/ReadVariableOp/Enter"
  input: "add_554"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_704"
  op: "ReadVariableOp"
  input: "Mul_884/ReadVariableOp/Enter"
  input: "^AssignVariableOp_481"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_482"
  op: "AssignVariableOp"
  input: "Mul_886/ReadVariableOp/Enter"
  input: "add_555"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_705"
  op: "ReadVariableOp"
  input: "Mul_886/ReadVariableOp/Enter"
  input: "^AssignVariableOp_482"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_889/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_889/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_889/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_889/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_889"
  op: "Mul"
  input: "Mul_889/x"
  input: "Mul_889/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_890/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_890"
  op: "Mul"
  input: "Mul_890/x"
  input: "CrossReplicaSum_161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_557"
  op: "AddV2"
  input: "Mul_889"
  input: "Mul_890"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_891/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_891/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_891/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_891/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_891"
  op: "Mul"
  input: "Mul_891/x"
  input: "Mul_891/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_161"
  op: "Square"
  input: "CrossReplicaSum_161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_892/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_892"
  op: "Mul"
  input: "Mul_892/x"
  input: "Square_161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_558"
  op: "AddV2"
  input: "Mul_891"
  input: "Mul_892"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_161"
  op: "Sqrt"
  input: "add_558"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_559/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_559"
  op: "AddV2"
  input: "Sqrt_161"
  input: "add_559/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_164"
  op: "RealDiv"
  input: "add_557"
  input: "add_559"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_706"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_893/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_893"
  op: "Mul"
  input: "mul_893/x"
  input: "ReadVariableOp_706"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_560"
  op: "AddV2"
  input: "truediv_164"
  input: "mul_893"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_894"
  op: "Mul"
  input: "mul_21"
  input: "add_560"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_707"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_165"
  op: "Sub"
  input: "ReadVariableOp_707"
  input: "mul_894"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_483"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_165"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_708"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_483"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_484"
  op: "AssignVariableOp"
  input: "Mul_889/ReadVariableOp/Enter"
  input: "add_557"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_709"
  op: "ReadVariableOp"
  input: "Mul_889/ReadVariableOp/Enter"
  input: "^AssignVariableOp_484"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_485"
  op: "AssignVariableOp"
  input: "Mul_891/ReadVariableOp/Enter"
  input: "add_558"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_710"
  op: "ReadVariableOp"
  input: "Mul_891/ReadVariableOp/Enter"
  input: "^AssignVariableOp_485"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_895/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_895/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_895/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_895/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_895"
  op: "Mul"
  input: "Mul_895/x"
  input: "Mul_895/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_896/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_896"
  op: "Mul"
  input: "Mul_896/x"
  input: "CrossReplicaSum_162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_561"
  op: "AddV2"
  input: "Mul_895"
  input: "Mul_896"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_897/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_897/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_897/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_897/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_897"
  op: "Mul"
  input: "Mul_897/x"
  input: "Mul_897/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_162"
  op: "Square"
  input: "CrossReplicaSum_162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_898/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_898"
  op: "Mul"
  input: "Mul_898/x"
  input: "Square_162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_562"
  op: "AddV2"
  input: "Mul_897"
  input: "Mul_898"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_162"
  op: "Sqrt"
  input: "add_562"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_563/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_563"
  op: "AddV2"
  input: "Sqrt_162"
  input: "add_563/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_165"
  op: "RealDiv"
  input: "add_561"
  input: "add_563"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_899"
  op: "Mul"
  input: "mul_21"
  input: "truediv_165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_711"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_166"
  op: "Sub"
  input: "ReadVariableOp_711"
  input: "mul_899"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_486"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_166"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_712"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_486"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_487"
  op: "AssignVariableOp"
  input: "Mul_895/ReadVariableOp/Enter"
  input: "add_561"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_713"
  op: "ReadVariableOp"
  input: "Mul_895/ReadVariableOp/Enter"
  input: "^AssignVariableOp_487"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_488"
  op: "AssignVariableOp"
  input: "Mul_897/ReadVariableOp/Enter"
  input: "add_562"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_714"
  op: "ReadVariableOp"
  input: "Mul_897/ReadVariableOp/Enter"
  input: "^AssignVariableOp_488"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_900/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_900/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_900/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_900/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_900"
  op: "Mul"
  input: "Mul_900/x"
  input: "Mul_900/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_901/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_901"
  op: "Mul"
  input: "Mul_901/x"
  input: "CrossReplicaSum_163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_564"
  op: "AddV2"
  input: "Mul_900"
  input: "Mul_901"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_902/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_902/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_902/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_902/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_902"
  op: "Mul"
  input: "Mul_902/x"
  input: "Mul_902/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_163"
  op: "Square"
  input: "CrossReplicaSum_163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_903/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_903"
  op: "Mul"
  input: "Mul_903/x"
  input: "Square_163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_565"
  op: "AddV2"
  input: "Mul_902"
  input: "Mul_903"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_163"
  op: "Sqrt"
  input: "add_565"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_566/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_566"
  op: "AddV2"
  input: "Sqrt_163"
  input: "add_566/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_166"
  op: "RealDiv"
  input: "add_564"
  input: "add_566"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_715"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_904/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_904"
  op: "Mul"
  input: "mul_904/x"
  input: "ReadVariableOp_715"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_567"
  op: "AddV2"
  input: "truediv_166"
  input: "mul_904"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_905"
  op: "Mul"
  input: "mul_21"
  input: "add_567"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_716"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_167"
  op: "Sub"
  input: "ReadVariableOp_716"
  input: "mul_905"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_489"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_167"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_717"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_489"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_490"
  op: "AssignVariableOp"
  input: "Mul_900/ReadVariableOp/Enter"
  input: "add_564"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_718"
  op: "ReadVariableOp"
  input: "Mul_900/ReadVariableOp/Enter"
  input: "^AssignVariableOp_490"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_491"
  op: "AssignVariableOp"
  input: "Mul_902/ReadVariableOp/Enter"
  input: "add_565"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_719"
  op: "ReadVariableOp"
  input: "Mul_902/ReadVariableOp/Enter"
  input: "^AssignVariableOp_491"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_m"
  input: "generator/encoder/layer_9/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_v"
  input: "generator/encoder/layer_9/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_906/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_906/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_906/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_906/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_906"
  op: "Mul"
  input: "Mul_906/x"
  input: "Mul_906/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_907/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_907"
  op: "Mul"
  input: "Mul_907/x"
  input: "CrossReplicaSum_164"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_568"
  op: "AddV2"
  input: "Mul_906"
  input: "Mul_907"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_908/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_908/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_908/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_908/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_908"
  op: "Mul"
  input: "Mul_908/x"
  input: "Mul_908/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_164"
  op: "Square"
  input: "CrossReplicaSum_164"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_909/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_909"
  op: "Mul"
  input: "Mul_909/x"
  input: "Square_164"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_569"
  op: "AddV2"
  input: "Mul_908"
  input: "Mul_909"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_164"
  op: "Sqrt"
  input: "add_569"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_570/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_570"
  op: "AddV2"
  input: "Sqrt_164"
  input: "add_570/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_167"
  op: "RealDiv"
  input: "add_568"
  input: "add_570"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_910"
  op: "Mul"
  input: "mul_21"
  input: "truediv_167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_720"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_168"
  op: "Sub"
  input: "ReadVariableOp_720"
  input: "mul_910"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_492"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_168"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_721"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_492"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_493"
  op: "AssignVariableOp"
  input: "Mul_906/ReadVariableOp/Enter"
  input: "add_568"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_722"
  op: "ReadVariableOp"
  input: "Mul_906/ReadVariableOp/Enter"
  input: "^AssignVariableOp_493"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_494"
  op: "AssignVariableOp"
  input: "Mul_908/ReadVariableOp/Enter"
  input: "add_569"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_723"
  op: "ReadVariableOp"
  input: "Mul_908/ReadVariableOp/Enter"
  input: "^AssignVariableOp_494"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_911/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_911/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_911/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_911/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_911"
  op: "Mul"
  input: "Mul_911/x"
  input: "Mul_911/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_912/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_912"
  op: "Mul"
  input: "Mul_912/x"
  input: "CrossReplicaSum_165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_571"
  op: "AddV2"
  input: "Mul_911"
  input: "Mul_912"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_913/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_913/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_913/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_913/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_913"
  op: "Mul"
  input: "Mul_913/x"
  input: "Mul_913/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_165"
  op: "Square"
  input: "CrossReplicaSum_165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_914/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_914"
  op: "Mul"
  input: "Mul_914/x"
  input: "Square_165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_572"
  op: "AddV2"
  input: "Mul_913"
  input: "Mul_914"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_165"
  op: "Sqrt"
  input: "add_572"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_573/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_573"
  op: "AddV2"
  input: "Sqrt_165"
  input: "add_573/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_168"
  op: "RealDiv"
  input: "add_571"
  input: "add_573"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_915"
  op: "Mul"
  input: "mul_21"
  input: "truediv_168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_724"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_169"
  op: "Sub"
  input: "ReadVariableOp_724"
  input: "mul_915"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_495"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_169"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_725"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_495"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_496"
  op: "AssignVariableOp"
  input: "Mul_911/ReadVariableOp/Enter"
  input: "add_571"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_726"
  op: "ReadVariableOp"
  input: "Mul_911/ReadVariableOp/Enter"
  input: "^AssignVariableOp_496"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_497"
  op: "AssignVariableOp"
  input: "Mul_913/ReadVariableOp/Enter"
  input: "add_572"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_727"
  op: "ReadVariableOp"
  input: "Mul_913/ReadVariableOp/Enter"
  input: "^AssignVariableOp_497"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_916/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_916/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_916/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_916/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_916"
  op: "Mul"
  input: "Mul_916/x"
  input: "Mul_916/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_917/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_917"
  op: "Mul"
  input: "Mul_917/x"
  input: "CrossReplicaSum_166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_574"
  op: "AddV2"
  input: "Mul_916"
  input: "Mul_917"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_918/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_918/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_918/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_918/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_918"
  op: "Mul"
  input: "Mul_918/x"
  input: "Mul_918/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_166"
  op: "Square"
  input: "CrossReplicaSum_166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_919/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_919"
  op: "Mul"
  input: "Mul_919/x"
  input: "Square_166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_575"
  op: "AddV2"
  input: "Mul_918"
  input: "Mul_919"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_166"
  op: "Sqrt"
  input: "add_575"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_576/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_576"
  op: "AddV2"
  input: "Sqrt_166"
  input: "add_576/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_169"
  op: "RealDiv"
  input: "add_574"
  input: "add_576"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_920"
  op: "Mul"
  input: "mul_21"
  input: "truediv_169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_728"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_170"
  op: "Sub"
  input: "ReadVariableOp_728"
  input: "mul_920"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_498"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_170"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_729"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_498"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_499"
  op: "AssignVariableOp"
  input: "Mul_916/ReadVariableOp/Enter"
  input: "add_574"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_730"
  op: "ReadVariableOp"
  input: "Mul_916/ReadVariableOp/Enter"
  input: "^AssignVariableOp_499"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_500"
  op: "AssignVariableOp"
  input: "Mul_918/ReadVariableOp/Enter"
  input: "add_575"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_731"
  op: "ReadVariableOp"
  input: "Mul_918/ReadVariableOp/Enter"
  input: "^AssignVariableOp_500"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_921/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_921/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_921/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_921/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_921"
  op: "Mul"
  input: "Mul_921/x"
  input: "Mul_921/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_922/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_922"
  op: "Mul"
  input: "Mul_922/x"
  input: "CrossReplicaSum_167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_577"
  op: "AddV2"
  input: "Mul_921"
  input: "Mul_922"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_923/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_923/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_923/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_923/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_923"
  op: "Mul"
  input: "Mul_923/x"
  input: "Mul_923/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_167"
  op: "Square"
  input: "CrossReplicaSum_167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_924/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_924"
  op: "Mul"
  input: "Mul_924/x"
  input: "Square_167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_578"
  op: "AddV2"
  input: "Mul_923"
  input: "Mul_924"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_167"
  op: "Sqrt"
  input: "add_578"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_579/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_579"
  op: "AddV2"
  input: "Sqrt_167"
  input: "add_579/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_170"
  op: "RealDiv"
  input: "add_577"
  input: "add_579"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_732"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_925/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_925"
  op: "Mul"
  input: "mul_925/x"
  input: "ReadVariableOp_732"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_580"
  op: "AddV2"
  input: "truediv_170"
  input: "mul_925"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_926"
  op: "Mul"
  input: "mul_21"
  input: "add_580"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_733"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_171"
  op: "Sub"
  input: "ReadVariableOp_733"
  input: "mul_926"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_501"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_171"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_734"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_501"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_502"
  op: "AssignVariableOp"
  input: "Mul_921/ReadVariableOp/Enter"
  input: "add_577"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_735"
  op: "ReadVariableOp"
  input: "Mul_921/ReadVariableOp/Enter"
  input: "^AssignVariableOp_502"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_503"
  op: "AssignVariableOp"
  input: "Mul_923/ReadVariableOp/Enter"
  input: "add_578"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_736"
  op: "ReadVariableOp"
  input: "Mul_923/ReadVariableOp/Enter"
  input: "^AssignVariableOp_503"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_927/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_927/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_927/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_927/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_927"
  op: "Mul"
  input: "Mul_927/x"
  input: "Mul_927/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_928/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_928"
  op: "Mul"
  input: "Mul_928/x"
  input: "CrossReplicaSum_168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_581"
  op: "AddV2"
  input: "Mul_927"
  input: "Mul_928"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_929/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_929/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_929/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_929/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_929"
  op: "Mul"
  input: "Mul_929/x"
  input: "Mul_929/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_168"
  op: "Square"
  input: "CrossReplicaSum_168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_930/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_930"
  op: "Mul"
  input: "Mul_930/x"
  input: "Square_168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_582"
  op: "AddV2"
  input: "Mul_929"
  input: "Mul_930"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_168"
  op: "Sqrt"
  input: "add_582"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_583/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_583"
  op: "AddV2"
  input: "Sqrt_168"
  input: "add_583/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_171"
  op: "RealDiv"
  input: "add_581"
  input: "add_583"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_931"
  op: "Mul"
  input: "mul_21"
  input: "truediv_171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_737"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_172"
  op: "Sub"
  input: "ReadVariableOp_737"
  input: "mul_931"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_504"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_172"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_738"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_504"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_505"
  op: "AssignVariableOp"
  input: "Mul_927/ReadVariableOp/Enter"
  input: "add_581"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_739"
  op: "ReadVariableOp"
  input: "Mul_927/ReadVariableOp/Enter"
  input: "^AssignVariableOp_505"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_506"
  op: "AssignVariableOp"
  input: "Mul_929/ReadVariableOp/Enter"
  input: "add_582"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_740"
  op: "ReadVariableOp"
  input: "Mul_929/ReadVariableOp/Enter"
  input: "^AssignVariableOp_506"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_932/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_932/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_932/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_932/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_932"
  op: "Mul"
  input: "Mul_932/x"
  input: "Mul_932/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_933/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_933"
  op: "Mul"
  input: "Mul_933/x"
  input: "CrossReplicaSum_169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_584"
  op: "AddV2"
  input: "Mul_932"
  input: "Mul_933"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_934/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_934/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_934/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_934/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_934"
  op: "Mul"
  input: "Mul_934/x"
  input: "Mul_934/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_169"
  op: "Square"
  input: "CrossReplicaSum_169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_935/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_935"
  op: "Mul"
  input: "Mul_935/x"
  input: "Square_169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_585"
  op: "AddV2"
  input: "Mul_934"
  input: "Mul_935"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_169"
  op: "Sqrt"
  input: "add_585"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_586/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_586"
  op: "AddV2"
  input: "Sqrt_169"
  input: "add_586/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_172"
  op: "RealDiv"
  input: "add_584"
  input: "add_586"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_741"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_936/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_936"
  op: "Mul"
  input: "mul_936/x"
  input: "ReadVariableOp_741"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_587"
  op: "AddV2"
  input: "truediv_172"
  input: "mul_936"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_937"
  op: "Mul"
  input: "mul_21"
  input: "add_587"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_742"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_173"
  op: "Sub"
  input: "ReadVariableOp_742"
  input: "mul_937"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_507"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_173"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_743"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_507"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_508"
  op: "AssignVariableOp"
  input: "Mul_932/ReadVariableOp/Enter"
  input: "add_584"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_744"
  op: "ReadVariableOp"
  input: "Mul_932/ReadVariableOp/Enter"
  input: "^AssignVariableOp_508"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_509"
  op: "AssignVariableOp"
  input: "Mul_934/ReadVariableOp/Enter"
  input: "add_585"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_745"
  op: "ReadVariableOp"
  input: "Mul_934/ReadVariableOp/Enter"
  input: "^AssignVariableOp_509"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_938/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_938/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_938/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_938/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_938"
  op: "Mul"
  input: "Mul_938/x"
  input: "Mul_938/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_939/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_939"
  op: "Mul"
  input: "Mul_939/x"
  input: "CrossReplicaSum_170"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_588"
  op: "AddV2"
  input: "Mul_938"
  input: "Mul_939"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_940/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_940/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_940/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_940/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_940"
  op: "Mul"
  input: "Mul_940/x"
  input: "Mul_940/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_170"
  op: "Square"
  input: "CrossReplicaSum_170"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_941/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_941"
  op: "Mul"
  input: "Mul_941/x"
  input: "Square_170"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_589"
  op: "AddV2"
  input: "Mul_940"
  input: "Mul_941"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_170"
  op: "Sqrt"
  input: "add_589"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_590/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_590"
  op: "AddV2"
  input: "Sqrt_170"
  input: "add_590/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_173"
  op: "RealDiv"
  input: "add_588"
  input: "add_590"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_942"
  op: "Mul"
  input: "mul_21"
  input: "truediv_173"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_746"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_174"
  op: "Sub"
  input: "ReadVariableOp_746"
  input: "mul_942"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_510"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_174"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_747"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_510"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_511"
  op: "AssignVariableOp"
  input: "Mul_938/ReadVariableOp/Enter"
  input: "add_588"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_748"
  op: "ReadVariableOp"
  input: "Mul_938/ReadVariableOp/Enter"
  input: "^AssignVariableOp_511"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_512"
  op: "AssignVariableOp"
  input: "Mul_940/ReadVariableOp/Enter"
  input: "add_589"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_749"
  op: "ReadVariableOp"
  input: "Mul_940/ReadVariableOp/Enter"
  input: "^AssignVariableOp_512"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_943/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_943/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_943/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_943/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_943"
  op: "Mul"
  input: "Mul_943/x"
  input: "Mul_943/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_944/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_944"
  op: "Mul"
  input: "Mul_944/x"
  input: "CrossReplicaSum_171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_591"
  op: "AddV2"
  input: "Mul_943"
  input: "Mul_944"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_945/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_945/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_945/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_945/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_945"
  op: "Mul"
  input: "Mul_945/x"
  input: "Mul_945/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_171"
  op: "Square"
  input: "CrossReplicaSum_171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_946/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_946"
  op: "Mul"
  input: "Mul_946/x"
  input: "Square_171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_592"
  op: "AddV2"
  input: "Mul_945"
  input: "Mul_946"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_171"
  op: "Sqrt"
  input: "add_592"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_593/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_593"
  op: "AddV2"
  input: "Sqrt_171"
  input: "add_593/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_174"
  op: "RealDiv"
  input: "add_591"
  input: "add_593"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_750"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_947/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_947"
  op: "Mul"
  input: "mul_947/x"
  input: "ReadVariableOp_750"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_594"
  op: "AddV2"
  input: "truediv_174"
  input: "mul_947"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_948"
  op: "Mul"
  input: "mul_21"
  input: "add_594"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_751"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_175"
  op: "Sub"
  input: "ReadVariableOp_751"
  input: "mul_948"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_513"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_175"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_752"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_513"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_514"
  op: "AssignVariableOp"
  input: "Mul_943/ReadVariableOp/Enter"
  input: "add_591"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_753"
  op: "ReadVariableOp"
  input: "Mul_943/ReadVariableOp/Enter"
  input: "^AssignVariableOp_514"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_515"
  op: "AssignVariableOp"
  input: "Mul_945/ReadVariableOp/Enter"
  input: "add_592"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_754"
  op: "ReadVariableOp"
  input: "Mul_945/ReadVariableOp/Enter"
  input: "^AssignVariableOp_515"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_949/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_949/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_949/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_949/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_949"
  op: "Mul"
  input: "Mul_949/x"
  input: "Mul_949/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_950/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_950"
  op: "Mul"
  input: "Mul_950/x"
  input: "CrossReplicaSum_172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_595"
  op: "AddV2"
  input: "Mul_949"
  input: "Mul_950"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_951/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_951/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_951/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_951/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_951"
  op: "Mul"
  input: "Mul_951/x"
  input: "Mul_951/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_172"
  op: "Square"
  input: "CrossReplicaSum_172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_952/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_952"
  op: "Mul"
  input: "Mul_952/x"
  input: "Square_172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_596"
  op: "AddV2"
  input: "Mul_951"
  input: "Mul_952"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_172"
  op: "Sqrt"
  input: "add_596"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_597/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_597"
  op: "AddV2"
  input: "Sqrt_172"
  input: "add_597/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_175"
  op: "RealDiv"
  input: "add_595"
  input: "add_597"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_953"
  op: "Mul"
  input: "mul_21"
  input: "truediv_175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_755"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_176"
  op: "Sub"
  input: "ReadVariableOp_755"
  input: "mul_953"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_516"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_176"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_756"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_516"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_517"
  op: "AssignVariableOp"
  input: "Mul_949/ReadVariableOp/Enter"
  input: "add_595"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_757"
  op: "ReadVariableOp"
  input: "Mul_949/ReadVariableOp/Enter"
  input: "^AssignVariableOp_517"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_518"
  op: "AssignVariableOp"
  input: "Mul_951/ReadVariableOp/Enter"
  input: "add_596"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_758"
  op: "ReadVariableOp"
  input: "Mul_951/ReadVariableOp/Enter"
  input: "^AssignVariableOp_518"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_954/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_954/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_954/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_954/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_954"
  op: "Mul"
  input: "Mul_954/x"
  input: "Mul_954/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_955/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_955"
  op: "Mul"
  input: "Mul_955/x"
  input: "CrossReplicaSum_173"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_598"
  op: "AddV2"
  input: "Mul_954"
  input: "Mul_955"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_956/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_956/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_956/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_956/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_956"
  op: "Mul"
  input: "Mul_956/x"
  input: "Mul_956/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_173"
  op: "Square"
  input: "CrossReplicaSum_173"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_957/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_957"
  op: "Mul"
  input: "Mul_957/x"
  input: "Square_173"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_599"
  op: "AddV2"
  input: "Mul_956"
  input: "Mul_957"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_173"
  op: "Sqrt"
  input: "add_599"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_600/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_600"
  op: "AddV2"
  input: "Sqrt_173"
  input: "add_600/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_176"
  op: "RealDiv"
  input: "add_598"
  input: "add_600"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_759"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_958/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_958"
  op: "Mul"
  input: "mul_958/x"
  input: "ReadVariableOp_759"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_601"
  op: "AddV2"
  input: "truediv_176"
  input: "mul_958"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_959"
  op: "Mul"
  input: "mul_21"
  input: "add_601"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_760"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_177"
  op: "Sub"
  input: "ReadVariableOp_760"
  input: "mul_959"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_519"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_177"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_761"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_519"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_520"
  op: "AssignVariableOp"
  input: "Mul_954/ReadVariableOp/Enter"
  input: "add_598"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_762"
  op: "ReadVariableOp"
  input: "Mul_954/ReadVariableOp/Enter"
  input: "^AssignVariableOp_520"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_521"
  op: "AssignVariableOp"
  input: "Mul_956/ReadVariableOp/Enter"
  input: "add_599"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_763"
  op: "ReadVariableOp"
  input: "Mul_956/ReadVariableOp/Enter"
  input: "^AssignVariableOp_521"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_960/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_960/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_960/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_960/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_960"
  op: "Mul"
  input: "Mul_960/x"
  input: "Mul_960/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_961/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_961"
  op: "Mul"
  input: "Mul_961/x"
  input: "CrossReplicaSum_174"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_602"
  op: "AddV2"
  input: "Mul_960"
  input: "Mul_961"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_962/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_962/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_962/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_962/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_962"
  op: "Mul"
  input: "Mul_962/x"
  input: "Mul_962/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_174"
  op: "Square"
  input: "CrossReplicaSum_174"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_963/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_963"
  op: "Mul"
  input: "Mul_963/x"
  input: "Square_174"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_603"
  op: "AddV2"
  input: "Mul_962"
  input: "Mul_963"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_174"
  op: "Sqrt"
  input: "add_603"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_604/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_604"
  op: "AddV2"
  input: "Sqrt_174"
  input: "add_604/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_177"
  op: "RealDiv"
  input: "add_602"
  input: "add_604"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_964"
  op: "Mul"
  input: "mul_21"
  input: "truediv_177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_764"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_178"
  op: "Sub"
  input: "ReadVariableOp_764"
  input: "mul_964"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_522"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_178"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_765"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_522"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_523"
  op: "AssignVariableOp"
  input: "Mul_960/ReadVariableOp/Enter"
  input: "add_602"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_766"
  op: "ReadVariableOp"
  input: "Mul_960/ReadVariableOp/Enter"
  input: "^AssignVariableOp_523"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_524"
  op: "AssignVariableOp"
  input: "Mul_962/ReadVariableOp/Enter"
  input: "add_603"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_767"
  op: "ReadVariableOp"
  input: "Mul_962/ReadVariableOp/Enter"
  input: "^AssignVariableOp_524"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_965/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_965/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_965/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_965/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_965"
  op: "Mul"
  input: "Mul_965/x"
  input: "Mul_965/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_966/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_966"
  op: "Mul"
  input: "Mul_966/x"
  input: "CrossReplicaSum_175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_605"
  op: "AddV2"
  input: "Mul_965"
  input: "Mul_966"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_967/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_967/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_967/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_967/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_967"
  op: "Mul"
  input: "Mul_967/x"
  input: "Mul_967/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_175"
  op: "Square"
  input: "CrossReplicaSum_175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_968/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_968"
  op: "Mul"
  input: "Mul_968/x"
  input: "Square_175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_606"
  op: "AddV2"
  input: "Mul_967"
  input: "Mul_968"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_175"
  op: "Sqrt"
  input: "add_606"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_607/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_607"
  op: "AddV2"
  input: "Sqrt_175"
  input: "add_607/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_178"
  op: "RealDiv"
  input: "add_605"
  input: "add_607"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_969"
  op: "Mul"
  input: "mul_21"
  input: "truediv_178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_768"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_179"
  op: "Sub"
  input: "ReadVariableOp_768"
  input: "mul_969"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_525"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_179"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_769"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_525"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_526"
  op: "AssignVariableOp"
  input: "Mul_965/ReadVariableOp/Enter"
  input: "add_605"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_770"
  op: "ReadVariableOp"
  input: "Mul_965/ReadVariableOp/Enter"
  input: "^AssignVariableOp_526"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_527"
  op: "AssignVariableOp"
  input: "Mul_967/ReadVariableOp/Enter"
  input: "add_606"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_771"
  op: "ReadVariableOp"
  input: "Mul_967/ReadVariableOp/Enter"
  input: "^AssignVariableOp_527"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_970/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_970/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_970/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_970/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_970"
  op: "Mul"
  input: "Mul_970/x"
  input: "Mul_970/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_971/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_971"
  op: "Mul"
  input: "Mul_971/x"
  input: "CrossReplicaSum_176"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_608"
  op: "AddV2"
  input: "Mul_970"
  input: "Mul_971"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_972/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_972/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_972/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_972/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_972"
  op: "Mul"
  input: "Mul_972/x"
  input: "Mul_972/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_176"
  op: "Square"
  input: "CrossReplicaSum_176"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_973/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_973"
  op: "Mul"
  input: "Mul_973/x"
  input: "Square_176"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_609"
  op: "AddV2"
  input: "Mul_972"
  input: "Mul_973"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_176"
  op: "Sqrt"
  input: "add_609"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_610/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_610"
  op: "AddV2"
  input: "Sqrt_176"
  input: "add_610/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_179"
  op: "RealDiv"
  input: "add_608"
  input: "add_610"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_974"
  op: "Mul"
  input: "mul_21"
  input: "truediv_179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_772"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_180"
  op: "Sub"
  input: "ReadVariableOp_772"
  input: "mul_974"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_528"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_180"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_773"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_528"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_529"
  op: "AssignVariableOp"
  input: "Mul_970/ReadVariableOp/Enter"
  input: "add_608"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_774"
  op: "ReadVariableOp"
  input: "Mul_970/ReadVariableOp/Enter"
  input: "^AssignVariableOp_529"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_530"
  op: "AssignVariableOp"
  input: "Mul_972/ReadVariableOp/Enter"
  input: "add_609"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_775"
  op: "ReadVariableOp"
  input: "Mul_972/ReadVariableOp/Enter"
  input: "^AssignVariableOp_530"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_975/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_975/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_975/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_975/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_975"
  op: "Mul"
  input: "Mul_975/x"
  input: "Mul_975/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_976/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_976"
  op: "Mul"
  input: "Mul_976/x"
  input: "CrossReplicaSum_177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_611"
  op: "AddV2"
  input: "Mul_975"
  input: "Mul_976"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_977/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_977/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_977/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_977/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_977"
  op: "Mul"
  input: "Mul_977/x"
  input: "Mul_977/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_177"
  op: "Square"
  input: "CrossReplicaSum_177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_978/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_978"
  op: "Mul"
  input: "Mul_978/x"
  input: "Square_177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_612"
  op: "AddV2"
  input: "Mul_977"
  input: "Mul_978"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_177"
  op: "Sqrt"
  input: "add_612"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_613/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_613"
  op: "AddV2"
  input: "Sqrt_177"
  input: "add_613/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_180"
  op: "RealDiv"
  input: "add_611"
  input: "add_613"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_776"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_979/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_979"
  op: "Mul"
  input: "mul_979/x"
  input: "ReadVariableOp_776"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_614"
  op: "AddV2"
  input: "truediv_180"
  input: "mul_979"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_980"
  op: "Mul"
  input: "mul_21"
  input: "add_614"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_777"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_181"
  op: "Sub"
  input: "ReadVariableOp_777"
  input: "mul_980"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_531"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_181"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_778"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_531"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_532"
  op: "AssignVariableOp"
  input: "Mul_975/ReadVariableOp/Enter"
  input: "add_611"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_779"
  op: "ReadVariableOp"
  input: "Mul_975/ReadVariableOp/Enter"
  input: "^AssignVariableOp_532"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_533"
  op: "AssignVariableOp"
  input: "Mul_977/ReadVariableOp/Enter"
  input: "add_612"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_780"
  op: "ReadVariableOp"
  input: "Mul_977/ReadVariableOp/Enter"
  input: "^AssignVariableOp_533"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_981/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_981/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_981/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_981/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_981"
  op: "Mul"
  input: "Mul_981/x"
  input: "Mul_981/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_982/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_982"
  op: "Mul"
  input: "Mul_982/x"
  input: "CrossReplicaSum_178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_615"
  op: "AddV2"
  input: "Mul_981"
  input: "Mul_982"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_983/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_983/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_983/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_983/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_983"
  op: "Mul"
  input: "Mul_983/x"
  input: "Mul_983/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_178"
  op: "Square"
  input: "CrossReplicaSum_178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_984/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_984"
  op: "Mul"
  input: "Mul_984/x"
  input: "Square_178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_616"
  op: "AddV2"
  input: "Mul_983"
  input: "Mul_984"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_178"
  op: "Sqrt"
  input: "add_616"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_617/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_617"
  op: "AddV2"
  input: "Sqrt_178"
  input: "add_617/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_181"
  op: "RealDiv"
  input: "add_615"
  input: "add_617"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_985"
  op: "Mul"
  input: "mul_21"
  input: "truediv_181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_781"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_182"
  op: "Sub"
  input: "ReadVariableOp_781"
  input: "mul_985"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_534"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_182"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_782"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_534"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_535"
  op: "AssignVariableOp"
  input: "Mul_981/ReadVariableOp/Enter"
  input: "add_615"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_783"
  op: "ReadVariableOp"
  input: "Mul_981/ReadVariableOp/Enter"
  input: "^AssignVariableOp_535"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_536"
  op: "AssignVariableOp"
  input: "Mul_983/ReadVariableOp/Enter"
  input: "add_616"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_784"
  op: "ReadVariableOp"
  input: "Mul_983/ReadVariableOp/Enter"
  input: "^AssignVariableOp_536"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_986/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_986/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_986/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_986/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_986"
  op: "Mul"
  input: "Mul_986/x"
  input: "Mul_986/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_987/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_987"
  op: "Mul"
  input: "Mul_987/x"
  input: "CrossReplicaSum_179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_618"
  op: "AddV2"
  input: "Mul_986"
  input: "Mul_987"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_988/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_988/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_988/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_988/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_988"
  op: "Mul"
  input: "Mul_988/x"
  input: "Mul_988/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_179"
  op: "Square"
  input: "CrossReplicaSum_179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_989/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_989"
  op: "Mul"
  input: "Mul_989/x"
  input: "Square_179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_619"
  op: "AddV2"
  input: "Mul_988"
  input: "Mul_989"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_179"
  op: "Sqrt"
  input: "add_619"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_620/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_620"
  op: "AddV2"
  input: "Sqrt_179"
  input: "add_620/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_182"
  op: "RealDiv"
  input: "add_618"
  input: "add_620"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_785"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_990/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_990"
  op: "Mul"
  input: "mul_990/x"
  input: "ReadVariableOp_785"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_621"
  op: "AddV2"
  input: "truediv_182"
  input: "mul_990"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_991"
  op: "Mul"
  input: "mul_21"
  input: "add_621"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_786"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_183"
  op: "Sub"
  input: "ReadVariableOp_786"
  input: "mul_991"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_537"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_183"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_787"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_537"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_538"
  op: "AssignVariableOp"
  input: "Mul_986/ReadVariableOp/Enter"
  input: "add_618"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_788"
  op: "ReadVariableOp"
  input: "Mul_986/ReadVariableOp/Enter"
  input: "^AssignVariableOp_538"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_539"
  op: "AssignVariableOp"
  input: "Mul_988/ReadVariableOp/Enter"
  input: "add_619"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_789"
  op: "ReadVariableOp"
  input: "Mul_988/ReadVariableOp/Enter"
  input: "^AssignVariableOp_539"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_m"
  input: "generator/encoder/layer_10/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_v"
  input: "generator/encoder/layer_10/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_992/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_992/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_992/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_992/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_992"
  op: "Mul"
  input: "Mul_992/x"
  input: "Mul_992/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_993/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_993"
  op: "Mul"
  input: "Mul_993/x"
  input: "CrossReplicaSum_180"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_622"
  op: "AddV2"
  input: "Mul_992"
  input: "Mul_993"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_994/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_994/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_994/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_994/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_994"
  op: "Mul"
  input: "Mul_994/x"
  input: "Mul_994/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_180"
  op: "Square"
  input: "CrossReplicaSum_180"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_995/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_995"
  op: "Mul"
  input: "Mul_995/x"
  input: "Square_180"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_623"
  op: "AddV2"
  input: "Mul_994"
  input: "Mul_995"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_180"
  op: "Sqrt"
  input: "add_623"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_624/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_624"
  op: "AddV2"
  input: "Sqrt_180"
  input: "add_624/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_183"
  op: "RealDiv"
  input: "add_622"
  input: "add_624"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_996"
  op: "Mul"
  input: "mul_21"
  input: "truediv_183"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_790"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_184"
  op: "Sub"
  input: "ReadVariableOp_790"
  input: "mul_996"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_540"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_184"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_791"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_540"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_541"
  op: "AssignVariableOp"
  input: "Mul_992/ReadVariableOp/Enter"
  input: "add_622"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_792"
  op: "ReadVariableOp"
  input: "Mul_992/ReadVariableOp/Enter"
  input: "^AssignVariableOp_541"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_542"
  op: "AssignVariableOp"
  input: "Mul_994/ReadVariableOp/Enter"
  input: "add_623"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_793"
  op: "ReadVariableOp"
  input: "Mul_994/ReadVariableOp/Enter"
  input: "^AssignVariableOp_542"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_997/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_997/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_997/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_997/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_997"
  op: "Mul"
  input: "Mul_997/x"
  input: "Mul_997/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_998/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_998"
  op: "Mul"
  input: "Mul_998/x"
  input: "CrossReplicaSum_181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_625"
  op: "AddV2"
  input: "Mul_997"
  input: "Mul_998"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_999/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_999/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_999/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_999/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_999"
  op: "Mul"
  input: "Mul_999/x"
  input: "Mul_999/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_181"
  op: "Square"
  input: "CrossReplicaSum_181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1000/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1000"
  op: "Mul"
  input: "Mul_1000/x"
  input: "Square_181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_626"
  op: "AddV2"
  input: "Mul_999"
  input: "Mul_1000"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_181"
  op: "Sqrt"
  input: "add_626"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_627/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_627"
  op: "AddV2"
  input: "Sqrt_181"
  input: "add_627/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_184"
  op: "RealDiv"
  input: "add_625"
  input: "add_627"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1001"
  op: "Mul"
  input: "mul_21"
  input: "truediv_184"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_794"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_185"
  op: "Sub"
  input: "ReadVariableOp_794"
  input: "mul_1001"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_543"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_185"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_795"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_543"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_544"
  op: "AssignVariableOp"
  input: "Mul_997/ReadVariableOp/Enter"
  input: "add_625"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_796"
  op: "ReadVariableOp"
  input: "Mul_997/ReadVariableOp/Enter"
  input: "^AssignVariableOp_544"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_545"
  op: "AssignVariableOp"
  input: "Mul_999/ReadVariableOp/Enter"
  input: "add_626"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_797"
  op: "ReadVariableOp"
  input: "Mul_999/ReadVariableOp/Enter"
  input: "^AssignVariableOp_545"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1002/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1002/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1002/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1002/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1002"
  op: "Mul"
  input: "Mul_1002/x"
  input: "Mul_1002/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1003/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1003"
  op: "Mul"
  input: "Mul_1003/x"
  input: "CrossReplicaSum_182"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_628"
  op: "AddV2"
  input: "Mul_1002"
  input: "Mul_1003"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1004/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1004/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1004/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1004/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1004"
  op: "Mul"
  input: "Mul_1004/x"
  input: "Mul_1004/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_182"
  op: "Square"
  input: "CrossReplicaSum_182"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1005/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1005"
  op: "Mul"
  input: "Mul_1005/x"
  input: "Square_182"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_629"
  op: "AddV2"
  input: "Mul_1004"
  input: "Mul_1005"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_182"
  op: "Sqrt"
  input: "add_629"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_630/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_630"
  op: "AddV2"
  input: "Sqrt_182"
  input: "add_630/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_185"
  op: "RealDiv"
  input: "add_628"
  input: "add_630"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1006"
  op: "Mul"
  input: "mul_21"
  input: "truediv_185"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_798"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_186"
  op: "Sub"
  input: "ReadVariableOp_798"
  input: "mul_1006"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_546"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_186"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_799"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_546"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_547"
  op: "AssignVariableOp"
  input: "Mul_1002/ReadVariableOp/Enter"
  input: "add_628"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_800"
  op: "ReadVariableOp"
  input: "Mul_1002/ReadVariableOp/Enter"
  input: "^AssignVariableOp_547"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_548"
  op: "AssignVariableOp"
  input: "Mul_1004/ReadVariableOp/Enter"
  input: "add_629"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_801"
  op: "ReadVariableOp"
  input: "Mul_1004/ReadVariableOp/Enter"
  input: "^AssignVariableOp_548"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1007/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1007/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1007/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1007/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1007"
  op: "Mul"
  input: "Mul_1007/x"
  input: "Mul_1007/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1008/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1008"
  op: "Mul"
  input: "Mul_1008/x"
  input: "CrossReplicaSum_183"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_631"
  op: "AddV2"
  input: "Mul_1007"
  input: "Mul_1008"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1009/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1009/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1009/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1009/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1009"
  op: "Mul"
  input: "Mul_1009/x"
  input: "Mul_1009/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_183"
  op: "Square"
  input: "CrossReplicaSum_183"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1010/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1010"
  op: "Mul"
  input: "Mul_1010/x"
  input: "Square_183"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_632"
  op: "AddV2"
  input: "Mul_1009"
  input: "Mul_1010"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_183"
  op: "Sqrt"
  input: "add_632"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_633/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_633"
  op: "AddV2"
  input: "Sqrt_183"
  input: "add_633/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_186"
  op: "RealDiv"
  input: "add_631"
  input: "add_633"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_802"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1011/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1011"
  op: "Mul"
  input: "mul_1011/x"
  input: "ReadVariableOp_802"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_634"
  op: "AddV2"
  input: "truediv_186"
  input: "mul_1011"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1012"
  op: "Mul"
  input: "mul_21"
  input: "add_634"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_803"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_187"
  op: "Sub"
  input: "ReadVariableOp_803"
  input: "mul_1012"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_549"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_187"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_804"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_549"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_550"
  op: "AssignVariableOp"
  input: "Mul_1007/ReadVariableOp/Enter"
  input: "add_631"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_805"
  op: "ReadVariableOp"
  input: "Mul_1007/ReadVariableOp/Enter"
  input: "^AssignVariableOp_550"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_551"
  op: "AssignVariableOp"
  input: "Mul_1009/ReadVariableOp/Enter"
  input: "add_632"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_806"
  op: "ReadVariableOp"
  input: "Mul_1009/ReadVariableOp/Enter"
  input: "^AssignVariableOp_551"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1013/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1013/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1013/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1013/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1013"
  op: "Mul"
  input: "Mul_1013/x"
  input: "Mul_1013/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1014/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1014"
  op: "Mul"
  input: "Mul_1014/x"
  input: "CrossReplicaSum_184"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_635"
  op: "AddV2"
  input: "Mul_1013"
  input: "Mul_1014"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1015/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1015/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1015/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1015/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1015"
  op: "Mul"
  input: "Mul_1015/x"
  input: "Mul_1015/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_184"
  op: "Square"
  input: "CrossReplicaSum_184"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1016/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1016"
  op: "Mul"
  input: "Mul_1016/x"
  input: "Square_184"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_636"
  op: "AddV2"
  input: "Mul_1015"
  input: "Mul_1016"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_184"
  op: "Sqrt"
  input: "add_636"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_637/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_637"
  op: "AddV2"
  input: "Sqrt_184"
  input: "add_637/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_187"
  op: "RealDiv"
  input: "add_635"
  input: "add_637"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1017"
  op: "Mul"
  input: "mul_21"
  input: "truediv_187"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_807"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_188"
  op: "Sub"
  input: "ReadVariableOp_807"
  input: "mul_1017"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_552"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_188"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_808"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_552"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_553"
  op: "AssignVariableOp"
  input: "Mul_1013/ReadVariableOp/Enter"
  input: "add_635"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_809"
  op: "ReadVariableOp"
  input: "Mul_1013/ReadVariableOp/Enter"
  input: "^AssignVariableOp_553"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_554"
  op: "AssignVariableOp"
  input: "Mul_1015/ReadVariableOp/Enter"
  input: "add_636"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_810"
  op: "ReadVariableOp"
  input: "Mul_1015/ReadVariableOp/Enter"
  input: "^AssignVariableOp_554"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1018/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1018/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1018/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1018/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1018"
  op: "Mul"
  input: "Mul_1018/x"
  input: "Mul_1018/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1019/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1019"
  op: "Mul"
  input: "Mul_1019/x"
  input: "CrossReplicaSum_185"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_638"
  op: "AddV2"
  input: "Mul_1018"
  input: "Mul_1019"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1020/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1020/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1020/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1020/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1020"
  op: "Mul"
  input: "Mul_1020/x"
  input: "Mul_1020/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_185"
  op: "Square"
  input: "CrossReplicaSum_185"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1021/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1021"
  op: "Mul"
  input: "Mul_1021/x"
  input: "Square_185"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_639"
  op: "AddV2"
  input: "Mul_1020"
  input: "Mul_1021"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_185"
  op: "Sqrt"
  input: "add_639"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_640/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_640"
  op: "AddV2"
  input: "Sqrt_185"
  input: "add_640/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_188"
  op: "RealDiv"
  input: "add_638"
  input: "add_640"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_811"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1022/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1022"
  op: "Mul"
  input: "mul_1022/x"
  input: "ReadVariableOp_811"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_641"
  op: "AddV2"
  input: "truediv_188"
  input: "mul_1022"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1023"
  op: "Mul"
  input: "mul_21"
  input: "add_641"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_812"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_189"
  op: "Sub"
  input: "ReadVariableOp_812"
  input: "mul_1023"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_555"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_189"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_813"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_555"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_556"
  op: "AssignVariableOp"
  input: "Mul_1018/ReadVariableOp/Enter"
  input: "add_638"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_814"
  op: "ReadVariableOp"
  input: "Mul_1018/ReadVariableOp/Enter"
  input: "^AssignVariableOp_556"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_557"
  op: "AssignVariableOp"
  input: "Mul_1020/ReadVariableOp/Enter"
  input: "add_639"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_815"
  op: "ReadVariableOp"
  input: "Mul_1020/ReadVariableOp/Enter"
  input: "^AssignVariableOp_557"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1024/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1024/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1024/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1024/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1024"
  op: "Mul"
  input: "Mul_1024/x"
  input: "Mul_1024/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1025/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1025"
  op: "Mul"
  input: "Mul_1025/x"
  input: "CrossReplicaSum_186"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_642"
  op: "AddV2"
  input: "Mul_1024"
  input: "Mul_1025"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1026/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1026/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1026/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1026/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1026"
  op: "Mul"
  input: "Mul_1026/x"
  input: "Mul_1026/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_186"
  op: "Square"
  input: "CrossReplicaSum_186"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1027/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1027"
  op: "Mul"
  input: "Mul_1027/x"
  input: "Square_186"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_643"
  op: "AddV2"
  input: "Mul_1026"
  input: "Mul_1027"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_186"
  op: "Sqrt"
  input: "add_643"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_644/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_644"
  op: "AddV2"
  input: "Sqrt_186"
  input: "add_644/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_189"
  op: "RealDiv"
  input: "add_642"
  input: "add_644"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1028"
  op: "Mul"
  input: "mul_21"
  input: "truediv_189"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_816"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_190"
  op: "Sub"
  input: "ReadVariableOp_816"
  input: "mul_1028"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_558"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_190"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_817"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_558"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_559"
  op: "AssignVariableOp"
  input: "Mul_1024/ReadVariableOp/Enter"
  input: "add_642"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_818"
  op: "ReadVariableOp"
  input: "Mul_1024/ReadVariableOp/Enter"
  input: "^AssignVariableOp_559"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_560"
  op: "AssignVariableOp"
  input: "Mul_1026/ReadVariableOp/Enter"
  input: "add_643"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_819"
  op: "ReadVariableOp"
  input: "Mul_1026/ReadVariableOp/Enter"
  input: "^AssignVariableOp_560"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1029/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1029/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1029/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1029/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1029"
  op: "Mul"
  input: "Mul_1029/x"
  input: "Mul_1029/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1030/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1030"
  op: "Mul"
  input: "Mul_1030/x"
  input: "CrossReplicaSum_187"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_645"
  op: "AddV2"
  input: "Mul_1029"
  input: "Mul_1030"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1031/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1031/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1031/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1031/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1031"
  op: "Mul"
  input: "Mul_1031/x"
  input: "Mul_1031/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_187"
  op: "Square"
  input: "CrossReplicaSum_187"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1032/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1032"
  op: "Mul"
  input: "Mul_1032/x"
  input: "Square_187"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_646"
  op: "AddV2"
  input: "Mul_1031"
  input: "Mul_1032"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_187"
  op: "Sqrt"
  input: "add_646"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_647/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_647"
  op: "AddV2"
  input: "Sqrt_187"
  input: "add_647/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_190"
  op: "RealDiv"
  input: "add_645"
  input: "add_647"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_820"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1033/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1033"
  op: "Mul"
  input: "mul_1033/x"
  input: "ReadVariableOp_820"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_648"
  op: "AddV2"
  input: "truediv_190"
  input: "mul_1033"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1034"
  op: "Mul"
  input: "mul_21"
  input: "add_648"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_821"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_191"
  op: "Sub"
  input: "ReadVariableOp_821"
  input: "mul_1034"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_561"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_191"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_822"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_561"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_562"
  op: "AssignVariableOp"
  input: "Mul_1029/ReadVariableOp/Enter"
  input: "add_645"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_823"
  op: "ReadVariableOp"
  input: "Mul_1029/ReadVariableOp/Enter"
  input: "^AssignVariableOp_562"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_563"
  op: "AssignVariableOp"
  input: "Mul_1031/ReadVariableOp/Enter"
  input: "add_646"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_824"
  op: "ReadVariableOp"
  input: "Mul_1031/ReadVariableOp/Enter"
  input: "^AssignVariableOp_563"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1035/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1035/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1035/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1035/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1035"
  op: "Mul"
  input: "Mul_1035/x"
  input: "Mul_1035/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1036/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1036"
  op: "Mul"
  input: "Mul_1036/x"
  input: "CrossReplicaSum_188"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_649"
  op: "AddV2"
  input: "Mul_1035"
  input: "Mul_1036"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1037/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1037/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1037/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1037/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1037"
  op: "Mul"
  input: "Mul_1037/x"
  input: "Mul_1037/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_188"
  op: "Square"
  input: "CrossReplicaSum_188"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1038/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1038"
  op: "Mul"
  input: "Mul_1038/x"
  input: "Square_188"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_650"
  op: "AddV2"
  input: "Mul_1037"
  input: "Mul_1038"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_188"
  op: "Sqrt"
  input: "add_650"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_651/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_651"
  op: "AddV2"
  input: "Sqrt_188"
  input: "add_651/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_191"
  op: "RealDiv"
  input: "add_649"
  input: "add_651"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1039"
  op: "Mul"
  input: "mul_21"
  input: "truediv_191"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_825"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_192"
  op: "Sub"
  input: "ReadVariableOp_825"
  input: "mul_1039"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_564"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_192"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_826"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_564"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_565"
  op: "AssignVariableOp"
  input: "Mul_1035/ReadVariableOp/Enter"
  input: "add_649"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_827"
  op: "ReadVariableOp"
  input: "Mul_1035/ReadVariableOp/Enter"
  input: "^AssignVariableOp_565"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_566"
  op: "AssignVariableOp"
  input: "Mul_1037/ReadVariableOp/Enter"
  input: "add_650"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_828"
  op: "ReadVariableOp"
  input: "Mul_1037/ReadVariableOp/Enter"
  input: "^AssignVariableOp_566"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1040/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1040/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1040/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1040/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1040"
  op: "Mul"
  input: "Mul_1040/x"
  input: "Mul_1040/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1041/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1041"
  op: "Mul"
  input: "Mul_1041/x"
  input: "CrossReplicaSum_189"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_652"
  op: "AddV2"
  input: "Mul_1040"
  input: "Mul_1041"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1042/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1042/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1042/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1042/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1042"
  op: "Mul"
  input: "Mul_1042/x"
  input: "Mul_1042/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_189"
  op: "Square"
  input: "CrossReplicaSum_189"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1043/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1043"
  op: "Mul"
  input: "Mul_1043/x"
  input: "Square_189"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_653"
  op: "AddV2"
  input: "Mul_1042"
  input: "Mul_1043"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_189"
  op: "Sqrt"
  input: "add_653"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_654/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_654"
  op: "AddV2"
  input: "Sqrt_189"
  input: "add_654/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_192"
  op: "RealDiv"
  input: "add_652"
  input: "add_654"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_829"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1044/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1044"
  op: "Mul"
  input: "mul_1044/x"
  input: "ReadVariableOp_829"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_655"
  op: "AddV2"
  input: "truediv_192"
  input: "mul_1044"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1045"
  op: "Mul"
  input: "mul_21"
  input: "add_655"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_830"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_193"
  op: "Sub"
  input: "ReadVariableOp_830"
  input: "mul_1045"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_567"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_193"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_831"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_567"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_568"
  op: "AssignVariableOp"
  input: "Mul_1040/ReadVariableOp/Enter"
  input: "add_652"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_832"
  op: "ReadVariableOp"
  input: "Mul_1040/ReadVariableOp/Enter"
  input: "^AssignVariableOp_568"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_569"
  op: "AssignVariableOp"
  input: "Mul_1042/ReadVariableOp/Enter"
  input: "add_653"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_833"
  op: "ReadVariableOp"
  input: "Mul_1042/ReadVariableOp/Enter"
  input: "^AssignVariableOp_569"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1046/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1046/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1046/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1046/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1046"
  op: "Mul"
  input: "Mul_1046/x"
  input: "Mul_1046/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1047/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1047"
  op: "Mul"
  input: "Mul_1047/x"
  input: "CrossReplicaSum_190"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_656"
  op: "AddV2"
  input: "Mul_1046"
  input: "Mul_1047"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1048/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1048/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1048/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1048/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1048"
  op: "Mul"
  input: "Mul_1048/x"
  input: "Mul_1048/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_190"
  op: "Square"
  input: "CrossReplicaSum_190"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1049/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1049"
  op: "Mul"
  input: "Mul_1049/x"
  input: "Square_190"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_657"
  op: "AddV2"
  input: "Mul_1048"
  input: "Mul_1049"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_190"
  op: "Sqrt"
  input: "add_657"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_658/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_658"
  op: "AddV2"
  input: "Sqrt_190"
  input: "add_658/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_193"
  op: "RealDiv"
  input: "add_656"
  input: "add_658"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1050"
  op: "Mul"
  input: "mul_21"
  input: "truediv_193"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_834"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_194"
  op: "Sub"
  input: "ReadVariableOp_834"
  input: "mul_1050"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_570"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_194"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_835"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_570"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_571"
  op: "AssignVariableOp"
  input: "Mul_1046/ReadVariableOp/Enter"
  input: "add_656"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_836"
  op: "ReadVariableOp"
  input: "Mul_1046/ReadVariableOp/Enter"
  input: "^AssignVariableOp_571"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_572"
  op: "AssignVariableOp"
  input: "Mul_1048/ReadVariableOp/Enter"
  input: "add_657"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_837"
  op: "ReadVariableOp"
  input: "Mul_1048/ReadVariableOp/Enter"
  input: "^AssignVariableOp_572"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1051/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1051/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1051/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1051/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1051"
  op: "Mul"
  input: "Mul_1051/x"
  input: "Mul_1051/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1052/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1052"
  op: "Mul"
  input: "Mul_1052/x"
  input: "CrossReplicaSum_191"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_659"
  op: "AddV2"
  input: "Mul_1051"
  input: "Mul_1052"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1053/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1053/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1053/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1053/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1053"
  op: "Mul"
  input: "Mul_1053/x"
  input: "Mul_1053/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_191"
  op: "Square"
  input: "CrossReplicaSum_191"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1054/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1054"
  op: "Mul"
  input: "Mul_1054/x"
  input: "Square_191"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_660"
  op: "AddV2"
  input: "Mul_1053"
  input: "Mul_1054"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_191"
  op: "Sqrt"
  input: "add_660"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_661/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_661"
  op: "AddV2"
  input: "Sqrt_191"
  input: "add_661/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_194"
  op: "RealDiv"
  input: "add_659"
  input: "add_661"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1055"
  op: "Mul"
  input: "mul_21"
  input: "truediv_194"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_838"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_195"
  op: "Sub"
  input: "ReadVariableOp_838"
  input: "mul_1055"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_573"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_195"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_839"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_573"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_574"
  op: "AssignVariableOp"
  input: "Mul_1051/ReadVariableOp/Enter"
  input: "add_659"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_840"
  op: "ReadVariableOp"
  input: "Mul_1051/ReadVariableOp/Enter"
  input: "^AssignVariableOp_574"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_575"
  op: "AssignVariableOp"
  input: "Mul_1053/ReadVariableOp/Enter"
  input: "add_660"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_841"
  op: "ReadVariableOp"
  input: "Mul_1053/ReadVariableOp/Enter"
  input: "^AssignVariableOp_575"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1056/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1056/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1056/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1056/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1056"
  op: "Mul"
  input: "Mul_1056/x"
  input: "Mul_1056/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1057/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1057"
  op: "Mul"
  input: "Mul_1057/x"
  input: "CrossReplicaSum_192"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_662"
  op: "AddV2"
  input: "Mul_1056"
  input: "Mul_1057"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1058/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1058/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1058/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1058/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1058"
  op: "Mul"
  input: "Mul_1058/x"
  input: "Mul_1058/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_192"
  op: "Square"
  input: "CrossReplicaSum_192"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1059/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1059"
  op: "Mul"
  input: "Mul_1059/x"
  input: "Square_192"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_663"
  op: "AddV2"
  input: "Mul_1058"
  input: "Mul_1059"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_192"
  op: "Sqrt"
  input: "add_663"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_664/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_664"
  op: "AddV2"
  input: "Sqrt_192"
  input: "add_664/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_195"
  op: "RealDiv"
  input: "add_662"
  input: "add_664"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1060"
  op: "Mul"
  input: "mul_21"
  input: "truediv_195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_842"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_196"
  op: "Sub"
  input: "ReadVariableOp_842"
  input: "mul_1060"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_576"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_196"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_843"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_576"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_577"
  op: "AssignVariableOp"
  input: "Mul_1056/ReadVariableOp/Enter"
  input: "add_662"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_844"
  op: "ReadVariableOp"
  input: "Mul_1056/ReadVariableOp/Enter"
  input: "^AssignVariableOp_577"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_578"
  op: "AssignVariableOp"
  input: "Mul_1058/ReadVariableOp/Enter"
  input: "add_663"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_845"
  op: "ReadVariableOp"
  input: "Mul_1058/ReadVariableOp/Enter"
  input: "^AssignVariableOp_578"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\004\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1061/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1061/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1061/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1061/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1061"
  op: "Mul"
  input: "Mul_1061/x"
  input: "Mul_1061/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1062/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1062"
  op: "Mul"
  input: "Mul_1062/x"
  input: "CrossReplicaSum_193"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_665"
  op: "AddV2"
  input: "Mul_1061"
  input: "Mul_1062"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1063/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1063/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1063/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1063/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1063"
  op: "Mul"
  input: "Mul_1063/x"
  input: "Mul_1063/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_193"
  op: "Square"
  input: "CrossReplicaSum_193"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1064/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1064"
  op: "Mul"
  input: "Mul_1064/x"
  input: "Square_193"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_666"
  op: "AddV2"
  input: "Mul_1063"
  input: "Mul_1064"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_193"
  op: "Sqrt"
  input: "add_666"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_667/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_667"
  op: "AddV2"
  input: "Sqrt_193"
  input: "add_667/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_196"
  op: "RealDiv"
  input: "add_665"
  input: "add_667"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_846"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1065/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1065"
  op: "Mul"
  input: "mul_1065/x"
  input: "ReadVariableOp_846"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_668"
  op: "AddV2"
  input: "truediv_196"
  input: "mul_1065"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1066"
  op: "Mul"
  input: "mul_21"
  input: "add_668"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_847"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_197"
  op: "Sub"
  input: "ReadVariableOp_847"
  input: "mul_1066"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_579"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_197"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_848"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_579"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_580"
  op: "AssignVariableOp"
  input: "Mul_1061/ReadVariableOp/Enter"
  input: "add_665"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_849"
  op: "ReadVariableOp"
  input: "Mul_1061/ReadVariableOp/Enter"
  input: "^AssignVariableOp_580"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_581"
  op: "AssignVariableOp"
  input: "Mul_1063/ReadVariableOp/Enter"
  input: "add_666"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_850"
  op: "ReadVariableOp"
  input: "Mul_1063/ReadVariableOp/Enter"
  input: "^AssignVariableOp_581"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1024
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1067/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1067/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1067/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1067/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1067"
  op: "Mul"
  input: "Mul_1067/x"
  input: "Mul_1067/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1068/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1068"
  op: "Mul"
  input: "Mul_1068/x"
  input: "CrossReplicaSum_194"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_669"
  op: "AddV2"
  input: "Mul_1067"
  input: "Mul_1068"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1069/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1069/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1069/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1069/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1069"
  op: "Mul"
  input: "Mul_1069/x"
  input: "Mul_1069/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_194"
  op: "Square"
  input: "CrossReplicaSum_194"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1070/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1070"
  op: "Mul"
  input: "Mul_1070/x"
  input: "Square_194"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_670"
  op: "AddV2"
  input: "Mul_1069"
  input: "Mul_1070"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_194"
  op: "Sqrt"
  input: "add_670"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_671/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_671"
  op: "AddV2"
  input: "Sqrt_194"
  input: "add_671/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_197"
  op: "RealDiv"
  input: "add_669"
  input: "add_671"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1071"
  op: "Mul"
  input: "mul_21"
  input: "truediv_197"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_851"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_198"
  op: "Sub"
  input: "ReadVariableOp_851"
  input: "mul_1071"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_582"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_198"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_852"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_582"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_583"
  op: "AssignVariableOp"
  input: "Mul_1067/ReadVariableOp/Enter"
  input: "add_669"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_853"
  op: "ReadVariableOp"
  input: "Mul_1067/ReadVariableOp/Enter"
  input: "^AssignVariableOp_583"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_584"
  op: "AssignVariableOp"
  input: "Mul_1069/ReadVariableOp/Enter"
  input: "add_670"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_854"
  op: "ReadVariableOp"
  input: "Mul_1069/ReadVariableOp/Enter"
  input: "^AssignVariableOp_584"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_m"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\004\000\000\000\001\000\000"
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1024
        }
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_v"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1072/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1072/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1072/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1072/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1072"
  op: "Mul"
  input: "Mul_1072/x"
  input: "Mul_1072/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1073/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1073"
  op: "Mul"
  input: "Mul_1073/x"
  input: "CrossReplicaSum_195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_672"
  op: "AddV2"
  input: "Mul_1072"
  input: "Mul_1073"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1074/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1074/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1074/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1074/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1074"
  op: "Mul"
  input: "Mul_1074/x"
  input: "Mul_1074/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_195"
  op: "Square"
  input: "CrossReplicaSum_195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1075/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1075"
  op: "Mul"
  input: "Mul_1075/x"
  input: "Square_195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_673"
  op: "AddV2"
  input: "Mul_1074"
  input: "Mul_1075"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_195"
  op: "Sqrt"
  input: "add_673"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_674/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_674"
  op: "AddV2"
  input: "Sqrt_195"
  input: "add_674/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_198"
  op: "RealDiv"
  input: "add_672"
  input: "add_674"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_855"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1076/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1076"
  op: "Mul"
  input: "mul_1076/x"
  input: "ReadVariableOp_855"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_675"
  op: "AddV2"
  input: "truediv_198"
  input: "mul_1076"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1077"
  op: "Mul"
  input: "mul_21"
  input: "add_675"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_856"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_199"
  op: "Sub"
  input: "ReadVariableOp_856"
  input: "mul_1077"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_585"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_199"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_857"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_585"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_586"
  op: "AssignVariableOp"
  input: "Mul_1072/ReadVariableOp/Enter"
  input: "add_672"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_858"
  op: "ReadVariableOp"
  input: "Mul_1072/ReadVariableOp/Enter"
  input: "^AssignVariableOp_586"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_587"
  op: "AssignVariableOp"
  input: "Mul_1074/ReadVariableOp/Enter"
  input: "add_673"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_859"
  op: "ReadVariableOp"
  input: "Mul_1074/ReadVariableOp/Enter"
  input: "^AssignVariableOp_587"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_m"
  input: "generator/encoder/layer_11/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_v"
  input: "generator/encoder/layer_11/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1078/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1078/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1078/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1078/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1078"
  op: "Mul"
  input: "Mul_1078/x"
  input: "Mul_1078/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1079/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1079"
  op: "Mul"
  input: "Mul_1079/x"
  input: "CrossReplicaSum_196"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_676"
  op: "AddV2"
  input: "Mul_1078"
  input: "Mul_1079"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1080/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1080/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1080/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1080/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1080"
  op: "Mul"
  input: "Mul_1080/x"
  input: "Mul_1080/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_196"
  op: "Square"
  input: "CrossReplicaSum_196"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1081/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1081"
  op: "Mul"
  input: "Mul_1081/x"
  input: "Square_196"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_677"
  op: "AddV2"
  input: "Mul_1080"
  input: "Mul_1081"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_196"
  op: "Sqrt"
  input: "add_677"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_678/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_678"
  op: "AddV2"
  input: "Sqrt_196"
  input: "add_678/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_199"
  op: "RealDiv"
  input: "add_676"
  input: "add_678"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1082"
  op: "Mul"
  input: "mul_21"
  input: "truediv_199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_860"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_200"
  op: "Sub"
  input: "ReadVariableOp_860"
  input: "mul_1082"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_588"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_200"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_861"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_588"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_589"
  op: "AssignVariableOp"
  input: "Mul_1078/ReadVariableOp/Enter"
  input: "add_676"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_862"
  op: "ReadVariableOp"
  input: "Mul_1078/ReadVariableOp/Enter"
  input: "^AssignVariableOp_589"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_590"
  op: "AssignVariableOp"
  input: "Mul_1080/ReadVariableOp/Enter"
  input: "add_677"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_863"
  op: "ReadVariableOp"
  input: "Mul_1080/ReadVariableOp/Enter"
  input: "^AssignVariableOp_590"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1083/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1083/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1083/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1083/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1083"
  op: "Mul"
  input: "Mul_1083/x"
  input: "Mul_1083/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1084/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1084"
  op: "Mul"
  input: "Mul_1084/x"
  input: "CrossReplicaSum_197"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_679"
  op: "AddV2"
  input: "Mul_1083"
  input: "Mul_1084"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1085/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1085/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1085/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1085/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1085"
  op: "Mul"
  input: "Mul_1085/x"
  input: "Mul_1085/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_197"
  op: "Square"
  input: "CrossReplicaSum_197"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1086/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1086"
  op: "Mul"
  input: "Mul_1086/x"
  input: "Square_197"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_680"
  op: "AddV2"
  input: "Mul_1085"
  input: "Mul_1086"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_197"
  op: "Sqrt"
  input: "add_680"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_681/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_681"
  op: "AddV2"
  input: "Sqrt_197"
  input: "add_681/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_200"
  op: "RealDiv"
  input: "add_679"
  input: "add_681"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1087"
  op: "Mul"
  input: "mul_21"
  input: "truediv_200"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_864"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_201"
  op: "Sub"
  input: "ReadVariableOp_864"
  input: "mul_1087"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_591"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_201"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_865"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_591"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_592"
  op: "AssignVariableOp"
  input: "Mul_1083/ReadVariableOp/Enter"
  input: "add_679"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_866"
  op: "ReadVariableOp"
  input: "Mul_1083/ReadVariableOp/Enter"
  input: "^AssignVariableOp_592"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_593"
  op: "AssignVariableOp"
  input: "Mul_1085/ReadVariableOp/Enter"
  input: "add_680"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_867"
  op: "ReadVariableOp"
  input: "Mul_1085/ReadVariableOp/Enter"
  input: "^AssignVariableOp_593"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 256
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1088/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1088/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1088/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1088/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1088"
  op: "Mul"
  input: "Mul_1088/x"
  input: "Mul_1088/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1089/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1089"
  op: "Mul"
  input: "Mul_1089/x"
  input: "CrossReplicaSum_198"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_682"
  op: "AddV2"
  input: "Mul_1088"
  input: "Mul_1089"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1090/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1090/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1090/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1090/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1090"
  op: "Mul"
  input: "Mul_1090/x"
  input: "Mul_1090/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_198"
  op: "Square"
  input: "CrossReplicaSum_198"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1091/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1091"
  op: "Mul"
  input: "Mul_1091/x"
  input: "Square_198"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_683"
  op: "AddV2"
  input: "Mul_1090"
  input: "Mul_1091"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_198"
  op: "Sqrt"
  input: "add_683"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_684/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_684"
  op: "AddV2"
  input: "Sqrt_198"
  input: "add_684/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_201"
  op: "RealDiv"
  input: "add_682"
  input: "add_684"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1092"
  op: "Mul"
  input: "mul_21"
  input: "truediv_201"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_868"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_202"
  op: "Sub"
  input: "ReadVariableOp_868"
  input: "mul_1092"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_594"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_202"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_869"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_594"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_595"
  op: "AssignVariableOp"
  input: "Mul_1088/ReadVariableOp/Enter"
  input: "add_682"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_870"
  op: "ReadVariableOp"
  input: "Mul_1088/ReadVariableOp/Enter"
  input: "^AssignVariableOp_595"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_596"
  op: "AssignVariableOp"
  input: "Mul_1090/ReadVariableOp/Enter"
  input: "add_683"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_871"
  op: "ReadVariableOp"
  input: "Mul_1090/ReadVariableOp/Enter"
  input: "^AssignVariableOp_596"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator_predictions/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator_predictions/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/dense/kernel/adam_m"
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/kernel/adam_m"
  input: "generator_predictions/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\001\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator_predictions/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator_predictions/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 256
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/dense/kernel/adam_v"
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/kernel/adam_v"
  input: "generator_predictions/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1093/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1093/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1093/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1093/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1093"
  op: "Mul"
  input: "Mul_1093/x"
  input: "Mul_1093/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1094/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1094"
  op: "Mul"
  input: "Mul_1094/x"
  input: "CrossReplicaSum_199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_685"
  op: "AddV2"
  input: "Mul_1093"
  input: "Mul_1094"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1095/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1095/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1095/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1095/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1095"
  op: "Mul"
  input: "Mul_1095/x"
  input: "Mul_1095/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_199"
  op: "Square"
  input: "CrossReplicaSum_199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1096/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1096"
  op: "Mul"
  input: "Mul_1096/x"
  input: "Square_199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_686"
  op: "AddV2"
  input: "Mul_1095"
  input: "Mul_1096"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_199"
  op: "Sqrt"
  input: "add_686"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_687/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_687"
  op: "AddV2"
  input: "Sqrt_199"
  input: "add_687/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_202"
  op: "RealDiv"
  input: "add_685"
  input: "add_687"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_872"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1097/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1097"
  op: "Mul"
  input: "mul_1097/x"
  input: "ReadVariableOp_872"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_688"
  op: "AddV2"
  input: "truediv_202"
  input: "mul_1097"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1098"
  op: "Mul"
  input: "mul_21"
  input: "add_688"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_873"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_203"
  op: "Sub"
  input: "ReadVariableOp_873"
  input: "mul_1098"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_597"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  input: "sub_203"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_874"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  input: "^AssignVariableOp_597"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_598"
  op: "AssignVariableOp"
  input: "Mul_1093/ReadVariableOp/Enter"
  input: "add_685"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_875"
  op: "ReadVariableOp"
  input: "Mul_1093/ReadVariableOp/Enter"
  input: "^AssignVariableOp_598"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_599"
  op: "AssignVariableOp"
  input: "Mul_1095/ReadVariableOp/Enter"
  input: "add_686"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_876"
  op: "ReadVariableOp"
  input: "Mul_1095/ReadVariableOp/Enter"
  input: "^AssignVariableOp_599"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/dense/bias/adam_m"
    }
  }
}
node {
  name: "generator_predictions/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/bias/adam_m"
  input: "generator_predictions/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/dense/bias/adam_v"
    }
  }
}
node {
  name: "generator_predictions/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/bias/adam_v"
  input: "generator_predictions/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1099/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1099/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1099/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1099/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1099"
  op: "Mul"
  input: "Mul_1099/x"
  input: "Mul_1099/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1100/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1100"
  op: "Mul"
  input: "Mul_1100/x"
  input: "CrossReplicaSum_200"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_689"
  op: "AddV2"
  input: "Mul_1099"
  input: "Mul_1100"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1101/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1101/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1101/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1101/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1101"
  op: "Mul"
  input: "Mul_1101/x"
  input: "Mul_1101/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_200"
  op: "Square"
  input: "CrossReplicaSum_200"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1102/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1102"
  op: "Mul"
  input: "Mul_1102/x"
  input: "Square_200"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_690"
  op: "AddV2"
  input: "Mul_1101"
  input: "Mul_1102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_200"
  op: "Sqrt"
  input: "add_690"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_691/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_691"
  op: "AddV2"
  input: "Sqrt_200"
  input: "add_691/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_203"
  op: "RealDiv"
  input: "add_689"
  input: "add_691"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1103"
  op: "Mul"
  input: "mul_21"
  input: "truediv_203"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_877"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_204"
  op: "Sub"
  input: "ReadVariableOp_877"
  input: "mul_1103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_600"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_204"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_878"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_600"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_601"
  op: "AssignVariableOp"
  input: "Mul_1099/ReadVariableOp/Enter"
  input: "add_689"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_879"
  op: "ReadVariableOp"
  input: "Mul_1099/ReadVariableOp/Enter"
  input: "^AssignVariableOp_601"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_602"
  op: "AssignVariableOp"
  input: "Mul_1101/ReadVariableOp/Enter"
  input: "add_690"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_880"
  op: "ReadVariableOp"
  input: "Mul_1101/ReadVariableOp/Enter"
  input: "^AssignVariableOp_602"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/gamma/adam_m"
  input: "generator_predictions/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/gamma/adam_v"
  input: "generator_predictions/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1104/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1104/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1104/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1104/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1104"
  op: "Mul"
  input: "Mul_1104/x"
  input: "Mul_1104/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1105/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1105"
  op: "Mul"
  input: "Mul_1105/x"
  input: "CrossReplicaSum_201"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_692"
  op: "AddV2"
  input: "Mul_1104"
  input: "Mul_1105"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1106/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1106/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1106/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1106/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1106"
  op: "Mul"
  input: "Mul_1106/x"
  input: "Mul_1106/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_201"
  op: "Square"
  input: "CrossReplicaSum_201"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1107/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1107"
  op: "Mul"
  input: "Mul_1107/x"
  input: "Square_201"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_693"
  op: "AddV2"
  input: "Mul_1106"
  input: "Mul_1107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_201"
  op: "Sqrt"
  input: "add_693"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_694/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_694"
  op: "AddV2"
  input: "Sqrt_201"
  input: "add_694/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_204"
  op: "RealDiv"
  input: "add_692"
  input: "add_694"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1108"
  op: "Mul"
  input: "mul_21"
  input: "truediv_204"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_881"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_205"
  op: "Sub"
  input: "ReadVariableOp_881"
  input: "mul_1108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_603"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_205"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_882"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_603"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_604"
  op: "AssignVariableOp"
  input: "Mul_1104/ReadVariableOp/Enter"
  input: "add_692"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_883"
  op: "ReadVariableOp"
  input: "Mul_1104/ReadVariableOp/Enter"
  input: "^AssignVariableOp_604"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_605"
  op: "AssignVariableOp"
  input: "Mul_1106/ReadVariableOp/Enter"
  input: "add_693"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_884"
  op: "ReadVariableOp"
  input: "Mul_1106/ReadVariableOp/Enter"
  input: "^AssignVariableOp_605"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/beta/adam_m"
  input: "generator_predictions/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/beta/adam_v"
  input: "generator_predictions/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1109/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1109/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1109/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1109/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1109"
  op: "Mul"
  input: "Mul_1109/x"
  input: "Mul_1109/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1110/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1110"
  op: "Mul"
  input: "Mul_1110/x"
  input: "CrossReplicaSum_202"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_695"
  op: "AddV2"
  input: "Mul_1109"
  input: "Mul_1110"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1111/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1111/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1111/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1111/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1111"
  op: "Mul"
  input: "Mul_1111/x"
  input: "Mul_1111/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_202"
  op: "Square"
  input: "CrossReplicaSum_202"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1112/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1112"
  op: "Mul"
  input: "Mul_1112/x"
  input: "Square_202"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_696"
  op: "AddV2"
  input: "Mul_1111"
  input: "Mul_1112"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_202"
  op: "Sqrt"
  input: "add_696"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_697/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_697"
  op: "AddV2"
  input: "Sqrt_202"
  input: "add_697/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_205"
  op: "RealDiv"
  input: "add_695"
  input: "add_697"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1113"
  op: "Mul"
  input: "mul_21"
  input: "truediv_205"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_885"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_206"
  op: "Sub"
  input: "ReadVariableOp_885"
  input: "mul_1113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_606"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_206"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_886"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_606"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_607"
  op: "AssignVariableOp"
  input: "Mul_1109/ReadVariableOp/Enter"
  input: "add_695"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_887"
  op: "ReadVariableOp"
  input: "Mul_1109/ReadVariableOp/Enter"
  input: "^AssignVariableOp_607"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_608"
  op: "AssignVariableOp"
  input: "Mul_1111/ReadVariableOp/Enter"
  input: "add_696"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_888"
  op: "ReadVariableOp"
  input: "Mul_1111/ReadVariableOp/Enter"
  input: "^AssignVariableOp_608"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 50265
      }
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "generator_predictions/output_bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "generator_predictions/output_bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 50265
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/output_bias/adam_m"
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/output_bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/output_bias/adam_m"
  input: "generator_predictions/output_bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/output_bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 50265
      }
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "generator_predictions/output_bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "generator_predictions/output_bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@generator_predictions/output_bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 50265
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "generator_predictions/output_bias/adam_v"
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "generator_predictions/output_bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "generator_predictions/output_bias/adam_v"
  input: "generator_predictions/output_bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "generator_predictions/output_bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/output_bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1114/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1114/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1114/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1114/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/output_bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1114"
  op: "Mul"
  input: "Mul_1114/x"
  input: "Mul_1114/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1115/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1115"
  op: "Mul"
  input: "Mul_1115/x"
  input: "CrossReplicaSum_203"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_698"
  op: "AddV2"
  input: "Mul_1114"
  input: "Mul_1115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1116/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1116/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1116/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1116/ReadVariableOp/Enter"
  op: "Enter"
  input: "generator_predictions/output_bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1116"
  op: "Mul"
  input: "Mul_1116/x"
  input: "Mul_1116/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_203"
  op: "Square"
  input: "CrossReplicaSum_203"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1117/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1117"
  op: "Mul"
  input: "Mul_1117/x"
  input: "Square_203"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_699"
  op: "AddV2"
  input: "Mul_1116"
  input: "Mul_1117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_203"
  op: "Sqrt"
  input: "add_699"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_700/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_700"
  op: "AddV2"
  input: "Sqrt_203"
  input: "add_700/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_206"
  op: "RealDiv"
  input: "add_698"
  input: "add_700"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1118"
  op: "Mul"
  input: "mul_21"
  input: "truediv_206"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_889"
  op: "ReadVariableOp"
  input: "generator_predictions/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_207"
  op: "Sub"
  input: "ReadVariableOp_889"
  input: "mul_1118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_609"
  op: "AssignVariableOp"
  input: "generator_predictions/BiasAdd/ReadVariableOp/Enter"
  input: "sub_207"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_890"
  op: "ReadVariableOp"
  input: "generator_predictions/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_609"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_610"
  op: "AssignVariableOp"
  input: "Mul_1114/ReadVariableOp/Enter"
  input: "add_698"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_891"
  op: "ReadVariableOp"
  input: "Mul_1114/ReadVariableOp/Enter"
  input: "^AssignVariableOp_610"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_611"
  op: "AssignVariableOp"
  input: "Mul_1116/ReadVariableOp/Enter"
  input: "add_699"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_892"
  op: "ReadVariableOp"
  input: "Mul_1116/ReadVariableOp/Enter"
  input: "^AssignVariableOp_611"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_3/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_m"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_3/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_v"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1119/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1119/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1119/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1119/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1119"
  op: "Mul"
  input: "Mul_1119/x"
  input: "Mul_1119/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1120/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1120"
  op: "Mul"
  input: "Mul_1120/x"
  input: "CrossReplicaSum_204"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_701"
  op: "AddV2"
  input: "Mul_1119"
  input: "Mul_1120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1121/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1121/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1121/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1121/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1121"
  op: "Mul"
  input: "Mul_1121/x"
  input: "Mul_1121/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_204"
  op: "Square"
  input: "CrossReplicaSum_204"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1122/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1122"
  op: "Mul"
  input: "Mul_1122/x"
  input: "Square_204"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_702"
  op: "AddV2"
  input: "Mul_1121"
  input: "Mul_1122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_204"
  op: "Sqrt"
  input: "add_702"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_703/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_703"
  op: "AddV2"
  input: "Sqrt_204"
  input: "add_703/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_207"
  op: "RealDiv"
  input: "add_701"
  input: "add_703"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1123"
  op: "Mul"
  input: "mul_21"
  input: "truediv_207"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_893"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_208"
  op: "Sub"
  input: "ReadVariableOp_893"
  input: "mul_1123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_612"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_208"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_894"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_612"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_613"
  op: "AssignVariableOp"
  input: "Mul_1119/ReadVariableOp/Enter"
  input: "add_701"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_895"
  op: "ReadVariableOp"
  input: "Mul_1119/ReadVariableOp/Enter"
  input: "^AssignVariableOp_613"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_614"
  op: "AssignVariableOp"
  input: "Mul_1121/ReadVariableOp/Enter"
  input: "add_702"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_896"
  op: "ReadVariableOp"
  input: "Mul_1121/ReadVariableOp/Enter"
  input: "^AssignVariableOp_614"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_3/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_m"
  input: "electra/embeddings_3/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/embeddings_3/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/embeddings_3/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_v"
  input: "electra/embeddings_3/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/embeddings_3/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1124/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1124/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1124/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1124/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_3/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1124"
  op: "Mul"
  input: "Mul_1124/x"
  input: "Mul_1124/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1125/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1125"
  op: "Mul"
  input: "Mul_1125/x"
  input: "CrossReplicaSum_205"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_704"
  op: "AddV2"
  input: "Mul_1124"
  input: "Mul_1125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1126/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1126/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1126/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1126/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/embeddings_3/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1126"
  op: "Mul"
  input: "Mul_1126/x"
  input: "Mul_1126/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_205"
  op: "Square"
  input: "CrossReplicaSum_205"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1127/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1127"
  op: "Mul"
  input: "Mul_1127/x"
  input: "Square_205"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_705"
  op: "AddV2"
  input: "Mul_1126"
  input: "Mul_1127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_205"
  op: "Sqrt"
  input: "add_705"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_706/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_706"
  op: "AddV2"
  input: "Sqrt_205"
  input: "add_706/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_208"
  op: "RealDiv"
  input: "add_704"
  input: "add_706"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1128"
  op: "Mul"
  input: "mul_21"
  input: "truediv_208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_897"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_209"
  op: "Sub"
  input: "ReadVariableOp_897"
  input: "mul_1128"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_615"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_209"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_898"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_615"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_616"
  op: "AssignVariableOp"
  input: "Mul_1124/ReadVariableOp/Enter"
  input: "add_704"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_899"
  op: "ReadVariableOp"
  input: "Mul_1124/ReadVariableOp/Enter"
  input: "^AssignVariableOp_616"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_617"
  op: "AssignVariableOp"
  input: "Mul_1126/ReadVariableOp/Enter"
  input: "add_705"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_900"
  op: "ReadVariableOp"
  input: "Mul_1126/ReadVariableOp/Enter"
  input: "^AssignVariableOp_617"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1129/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1129/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1129/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1129/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1129"
  op: "Mul"
  input: "Mul_1129/x"
  input: "Mul_1129/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1130/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1130"
  op: "Mul"
  input: "Mul_1130/x"
  input: "CrossReplicaSum_206"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_707"
  op: "AddV2"
  input: "Mul_1129"
  input: "Mul_1130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1131/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1131/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1131/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1131/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1131"
  op: "Mul"
  input: "Mul_1131/x"
  input: "Mul_1131/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_206"
  op: "Square"
  input: "CrossReplicaSum_206"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1132/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1132"
  op: "Mul"
  input: "Mul_1132/x"
  input: "Square_206"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_708"
  op: "AddV2"
  input: "Mul_1131"
  input: "Mul_1132"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_206"
  op: "Sqrt"
  input: "add_708"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_709/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_709"
  op: "AddV2"
  input: "Sqrt_206"
  input: "add_709/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_209"
  op: "RealDiv"
  input: "add_707"
  input: "add_709"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_901"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1133/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1133"
  op: "Mul"
  input: "mul_1133/x"
  input: "ReadVariableOp_901"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_710"
  op: "AddV2"
  input: "truediv_209"
  input: "mul_1133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1134"
  op: "Mul"
  input: "mul_21"
  input: "add_710"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_902"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_210"
  op: "Sub"
  input: "ReadVariableOp_902"
  input: "mul_1134"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_618"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_210"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_903"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_618"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_619"
  op: "AssignVariableOp"
  input: "Mul_1129/ReadVariableOp/Enter"
  input: "add_707"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_904"
  op: "ReadVariableOp"
  input: "Mul_1129/ReadVariableOp/Enter"
  input: "^AssignVariableOp_619"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_620"
  op: "AssignVariableOp"
  input: "Mul_1131/ReadVariableOp/Enter"
  input: "add_708"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_905"
  op: "ReadVariableOp"
  input: "Mul_1131/ReadVariableOp/Enter"
  input: "^AssignVariableOp_620"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1135/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1135/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1135/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1135/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1135"
  op: "Mul"
  input: "Mul_1135/x"
  input: "Mul_1135/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1136/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1136"
  op: "Mul"
  input: "Mul_1136/x"
  input: "CrossReplicaSum_207"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_711"
  op: "AddV2"
  input: "Mul_1135"
  input: "Mul_1136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1137/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1137/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1137/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1137/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1137"
  op: "Mul"
  input: "Mul_1137/x"
  input: "Mul_1137/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_207"
  op: "Square"
  input: "CrossReplicaSum_207"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1138/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1138"
  op: "Mul"
  input: "Mul_1138/x"
  input: "Square_207"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_712"
  op: "AddV2"
  input: "Mul_1137"
  input: "Mul_1138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_207"
  op: "Sqrt"
  input: "add_712"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_713/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_713"
  op: "AddV2"
  input: "Sqrt_207"
  input: "add_713/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_210"
  op: "RealDiv"
  input: "add_711"
  input: "add_713"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1139"
  op: "Mul"
  input: "mul_21"
  input: "truediv_210"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_906"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_211"
  op: "Sub"
  input: "ReadVariableOp_906"
  input: "mul_1139"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_621"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_211"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_907"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_621"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_622"
  op: "AssignVariableOp"
  input: "Mul_1135/ReadVariableOp/Enter"
  input: "add_711"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_908"
  op: "ReadVariableOp"
  input: "Mul_1135/ReadVariableOp/Enter"
  input: "^AssignVariableOp_622"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_623"
  op: "AssignVariableOp"
  input: "Mul_1137/ReadVariableOp/Enter"
  input: "add_712"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_909"
  op: "ReadVariableOp"
  input: "Mul_1137/ReadVariableOp/Enter"
  input: "^AssignVariableOp_623"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1140/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1140/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1140/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1140/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1140"
  op: "Mul"
  input: "Mul_1140/x"
  input: "Mul_1140/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1141/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1141"
  op: "Mul"
  input: "Mul_1141/x"
  input: "CrossReplicaSum_208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_714"
  op: "AddV2"
  input: "Mul_1140"
  input: "Mul_1141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1142/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1142/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1142/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1142/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1142"
  op: "Mul"
  input: "Mul_1142/x"
  input: "Mul_1142/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_208"
  op: "Square"
  input: "CrossReplicaSum_208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1143/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1143"
  op: "Mul"
  input: "Mul_1143/x"
  input: "Square_208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_715"
  op: "AddV2"
  input: "Mul_1142"
  input: "Mul_1143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_208"
  op: "Sqrt"
  input: "add_715"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_716/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_716"
  op: "AddV2"
  input: "Sqrt_208"
  input: "add_716/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_211"
  op: "RealDiv"
  input: "add_714"
  input: "add_716"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_910"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1144/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1144"
  op: "Mul"
  input: "mul_1144/x"
  input: "ReadVariableOp_910"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_717"
  op: "AddV2"
  input: "truediv_211"
  input: "mul_1144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1145"
  op: "Mul"
  input: "mul_21"
  input: "add_717"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_911"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_212"
  op: "Sub"
  input: "ReadVariableOp_911"
  input: "mul_1145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_624"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_212"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_912"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_624"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_625"
  op: "AssignVariableOp"
  input: "Mul_1140/ReadVariableOp/Enter"
  input: "add_714"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_913"
  op: "ReadVariableOp"
  input: "Mul_1140/ReadVariableOp/Enter"
  input: "^AssignVariableOp_625"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_626"
  op: "AssignVariableOp"
  input: "Mul_1142/ReadVariableOp/Enter"
  input: "add_715"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_914"
  op: "ReadVariableOp"
  input: "Mul_1142/ReadVariableOp/Enter"
  input: "^AssignVariableOp_626"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1146/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1146/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1146/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1146/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1146"
  op: "Mul"
  input: "Mul_1146/x"
  input: "Mul_1146/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1147/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1147"
  op: "Mul"
  input: "Mul_1147/x"
  input: "CrossReplicaSum_209"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_718"
  op: "AddV2"
  input: "Mul_1146"
  input: "Mul_1147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1148/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1148/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1148/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1148/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1148"
  op: "Mul"
  input: "Mul_1148/x"
  input: "Mul_1148/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_209"
  op: "Square"
  input: "CrossReplicaSum_209"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1149/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1149"
  op: "Mul"
  input: "Mul_1149/x"
  input: "Square_209"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_719"
  op: "AddV2"
  input: "Mul_1148"
  input: "Mul_1149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_209"
  op: "Sqrt"
  input: "add_719"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_720/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_720"
  op: "AddV2"
  input: "Sqrt_209"
  input: "add_720/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_212"
  op: "RealDiv"
  input: "add_718"
  input: "add_720"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1150"
  op: "Mul"
  input: "mul_21"
  input: "truediv_212"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_915"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_213"
  op: "Sub"
  input: "ReadVariableOp_915"
  input: "mul_1150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_627"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_213"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_916"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_627"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_628"
  op: "AssignVariableOp"
  input: "Mul_1146/ReadVariableOp/Enter"
  input: "add_718"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_917"
  op: "ReadVariableOp"
  input: "Mul_1146/ReadVariableOp/Enter"
  input: "^AssignVariableOp_628"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_629"
  op: "AssignVariableOp"
  input: "Mul_1148/ReadVariableOp/Enter"
  input: "add_719"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_918"
  op: "ReadVariableOp"
  input: "Mul_1148/ReadVariableOp/Enter"
  input: "^AssignVariableOp_629"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1151/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1151/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1151/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1151/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1151"
  op: "Mul"
  input: "Mul_1151/x"
  input: "Mul_1151/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1152/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1152"
  op: "Mul"
  input: "Mul_1152/x"
  input: "CrossReplicaSum_210"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_721"
  op: "AddV2"
  input: "Mul_1151"
  input: "Mul_1152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1153/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1153/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1153/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1153/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1153"
  op: "Mul"
  input: "Mul_1153/x"
  input: "Mul_1153/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_210"
  op: "Square"
  input: "CrossReplicaSum_210"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1154/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1154"
  op: "Mul"
  input: "Mul_1154/x"
  input: "Square_210"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_722"
  op: "AddV2"
  input: "Mul_1153"
  input: "Mul_1154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_210"
  op: "Sqrt"
  input: "add_722"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_723/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_723"
  op: "AddV2"
  input: "Sqrt_210"
  input: "add_723/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_213"
  op: "RealDiv"
  input: "add_721"
  input: "add_723"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_919"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1155/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1155"
  op: "Mul"
  input: "mul_1155/x"
  input: "ReadVariableOp_919"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_724"
  op: "AddV2"
  input: "truediv_213"
  input: "mul_1155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1156"
  op: "Mul"
  input: "mul_21"
  input: "add_724"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_920"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_214"
  op: "Sub"
  input: "ReadVariableOp_920"
  input: "mul_1156"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_630"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_214"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_921"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_630"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_631"
  op: "AssignVariableOp"
  input: "Mul_1151/ReadVariableOp/Enter"
  input: "add_721"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_922"
  op: "ReadVariableOp"
  input: "Mul_1151/ReadVariableOp/Enter"
  input: "^AssignVariableOp_631"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_632"
  op: "AssignVariableOp"
  input: "Mul_1153/ReadVariableOp/Enter"
  input: "add_722"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_923"
  op: "ReadVariableOp"
  input: "Mul_1153/ReadVariableOp/Enter"
  input: "^AssignVariableOp_632"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1157/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1157/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1157/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1157/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1157"
  op: "Mul"
  input: "Mul_1157/x"
  input: "Mul_1157/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1158/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1158"
  op: "Mul"
  input: "Mul_1158/x"
  input: "CrossReplicaSum_211"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_725"
  op: "AddV2"
  input: "Mul_1157"
  input: "Mul_1158"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1159/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1159/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1159/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1159/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1159"
  op: "Mul"
  input: "Mul_1159/x"
  input: "Mul_1159/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_211"
  op: "Square"
  input: "CrossReplicaSum_211"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1160/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1160"
  op: "Mul"
  input: "Mul_1160/x"
  input: "Square_211"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_726"
  op: "AddV2"
  input: "Mul_1159"
  input: "Mul_1160"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_211"
  op: "Sqrt"
  input: "add_726"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_727/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_727"
  op: "AddV2"
  input: "Sqrt_211"
  input: "add_727/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_214"
  op: "RealDiv"
  input: "add_725"
  input: "add_727"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1161"
  op: "Mul"
  input: "mul_21"
  input: "truediv_214"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_924"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_215"
  op: "Sub"
  input: "ReadVariableOp_924"
  input: "mul_1161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_633"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_215"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_925"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_633"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_634"
  op: "AssignVariableOp"
  input: "Mul_1157/ReadVariableOp/Enter"
  input: "add_725"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_926"
  op: "ReadVariableOp"
  input: "Mul_1157/ReadVariableOp/Enter"
  input: "^AssignVariableOp_634"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_635"
  op: "AssignVariableOp"
  input: "Mul_1159/ReadVariableOp/Enter"
  input: "add_726"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_927"
  op: "ReadVariableOp"
  input: "Mul_1159/ReadVariableOp/Enter"
  input: "^AssignVariableOp_635"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1162/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1162/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1162/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1162/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1162"
  op: "Mul"
  input: "Mul_1162/x"
  input: "Mul_1162/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1163/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1163"
  op: "Mul"
  input: "Mul_1163/x"
  input: "CrossReplicaSum_212"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_728"
  op: "AddV2"
  input: "Mul_1162"
  input: "Mul_1163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1164/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1164/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1164/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1164/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1164"
  op: "Mul"
  input: "Mul_1164/x"
  input: "Mul_1164/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_212"
  op: "Square"
  input: "CrossReplicaSum_212"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1165/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1165"
  op: "Mul"
  input: "Mul_1165/x"
  input: "Square_212"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_729"
  op: "AddV2"
  input: "Mul_1164"
  input: "Mul_1165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_212"
  op: "Sqrt"
  input: "add_729"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_730/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_730"
  op: "AddV2"
  input: "Sqrt_212"
  input: "add_730/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_215"
  op: "RealDiv"
  input: "add_728"
  input: "add_730"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_928"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1166/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1166"
  op: "Mul"
  input: "mul_1166/x"
  input: "ReadVariableOp_928"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_731"
  op: "AddV2"
  input: "truediv_215"
  input: "mul_1166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1167"
  op: "Mul"
  input: "mul_21"
  input: "add_731"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_929"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_216"
  op: "Sub"
  input: "ReadVariableOp_929"
  input: "mul_1167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_636"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_216"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_930"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_636"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_637"
  op: "AssignVariableOp"
  input: "Mul_1162/ReadVariableOp/Enter"
  input: "add_728"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_931"
  op: "ReadVariableOp"
  input: "Mul_1162/ReadVariableOp/Enter"
  input: "^AssignVariableOp_637"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_638"
  op: "AssignVariableOp"
  input: "Mul_1164/ReadVariableOp/Enter"
  input: "add_729"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_932"
  op: "ReadVariableOp"
  input: "Mul_1164/ReadVariableOp/Enter"
  input: "^AssignVariableOp_638"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1168/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1168/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1168/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1168/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1168"
  op: "Mul"
  input: "Mul_1168/x"
  input: "Mul_1168/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1169/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1169"
  op: "Mul"
  input: "Mul_1169/x"
  input: "CrossReplicaSum_213"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_732"
  op: "AddV2"
  input: "Mul_1168"
  input: "Mul_1169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1170/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1170/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1170/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1170/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1170"
  op: "Mul"
  input: "Mul_1170/x"
  input: "Mul_1170/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_213"
  op: "Square"
  input: "CrossReplicaSum_213"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1171/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1171"
  op: "Mul"
  input: "Mul_1171/x"
  input: "Square_213"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_733"
  op: "AddV2"
  input: "Mul_1170"
  input: "Mul_1171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_213"
  op: "Sqrt"
  input: "add_733"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_734/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_734"
  op: "AddV2"
  input: "Sqrt_213"
  input: "add_734/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_216"
  op: "RealDiv"
  input: "add_732"
  input: "add_734"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1172"
  op: "Mul"
  input: "mul_21"
  input: "truediv_216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_933"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_217"
  op: "Sub"
  input: "ReadVariableOp_933"
  input: "mul_1172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_639"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_217"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_934"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_639"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_640"
  op: "AssignVariableOp"
  input: "Mul_1168/ReadVariableOp/Enter"
  input: "add_732"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_935"
  op: "ReadVariableOp"
  input: "Mul_1168/ReadVariableOp/Enter"
  input: "^AssignVariableOp_640"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_641"
  op: "AssignVariableOp"
  input: "Mul_1170/ReadVariableOp/Enter"
  input: "add_733"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_936"
  op: "ReadVariableOp"
  input: "Mul_1170/ReadVariableOp/Enter"
  input: "^AssignVariableOp_641"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1173/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1173/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1173/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1173/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1173"
  op: "Mul"
  input: "Mul_1173/x"
  input: "Mul_1173/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1174/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1174"
  op: "Mul"
  input: "Mul_1174/x"
  input: "CrossReplicaSum_214"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_735"
  op: "AddV2"
  input: "Mul_1173"
  input: "Mul_1174"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1175/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1175/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1175/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1175/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1175"
  op: "Mul"
  input: "Mul_1175/x"
  input: "Mul_1175/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_214"
  op: "Square"
  input: "CrossReplicaSum_214"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1176/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1176"
  op: "Mul"
  input: "Mul_1176/x"
  input: "Square_214"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_736"
  op: "AddV2"
  input: "Mul_1175"
  input: "Mul_1176"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_214"
  op: "Sqrt"
  input: "add_736"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_737/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_737"
  op: "AddV2"
  input: "Sqrt_214"
  input: "add_737/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_217"
  op: "RealDiv"
  input: "add_735"
  input: "add_737"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1177"
  op: "Mul"
  input: "mul_21"
  input: "truediv_217"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_937"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_218"
  op: "Sub"
  input: "ReadVariableOp_937"
  input: "mul_1177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_642"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_218"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_938"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_642"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_643"
  op: "AssignVariableOp"
  input: "Mul_1173/ReadVariableOp/Enter"
  input: "add_735"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_939"
  op: "ReadVariableOp"
  input: "Mul_1173/ReadVariableOp/Enter"
  input: "^AssignVariableOp_643"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_644"
  op: "AssignVariableOp"
  input: "Mul_1175/ReadVariableOp/Enter"
  input: "add_736"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_940"
  op: "ReadVariableOp"
  input: "Mul_1175/ReadVariableOp/Enter"
  input: "^AssignVariableOp_644"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1178/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1178/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1178/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1178/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1178"
  op: "Mul"
  input: "Mul_1178/x"
  input: "Mul_1178/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1179/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1179"
  op: "Mul"
  input: "Mul_1179/x"
  input: "CrossReplicaSum_215"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_738"
  op: "AddV2"
  input: "Mul_1178"
  input: "Mul_1179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1180/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1180/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1180/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1180/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1180"
  op: "Mul"
  input: "Mul_1180/x"
  input: "Mul_1180/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_215"
  op: "Square"
  input: "CrossReplicaSum_215"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1181/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1181"
  op: "Mul"
  input: "Mul_1181/x"
  input: "Square_215"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_739"
  op: "AddV2"
  input: "Mul_1180"
  input: "Mul_1181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_215"
  op: "Sqrt"
  input: "add_739"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_740/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_740"
  op: "AddV2"
  input: "Sqrt_215"
  input: "add_740/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_218"
  op: "RealDiv"
  input: "add_738"
  input: "add_740"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1182"
  op: "Mul"
  input: "mul_21"
  input: "truediv_218"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_941"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_219"
  op: "Sub"
  input: "ReadVariableOp_941"
  input: "mul_1182"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_645"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_219"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_942"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_645"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_646"
  op: "AssignVariableOp"
  input: "Mul_1178/ReadVariableOp/Enter"
  input: "add_738"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_943"
  op: "ReadVariableOp"
  input: "Mul_1178/ReadVariableOp/Enter"
  input: "^AssignVariableOp_646"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_647"
  op: "AssignVariableOp"
  input: "Mul_1180/ReadVariableOp/Enter"
  input: "add_739"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_944"
  op: "ReadVariableOp"
  input: "Mul_1180/ReadVariableOp/Enter"
  input: "^AssignVariableOp_647"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1183/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1183/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1183/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1183/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1183"
  op: "Mul"
  input: "Mul_1183/x"
  input: "Mul_1183/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1184/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1184"
  op: "Mul"
  input: "Mul_1184/x"
  input: "CrossReplicaSum_216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_741"
  op: "AddV2"
  input: "Mul_1183"
  input: "Mul_1184"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1185/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1185/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1185/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1185/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1185"
  op: "Mul"
  input: "Mul_1185/x"
  input: "Mul_1185/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_216"
  op: "Square"
  input: "CrossReplicaSum_216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1186/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1186"
  op: "Mul"
  input: "Mul_1186/x"
  input: "Square_216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_742"
  op: "AddV2"
  input: "Mul_1185"
  input: "Mul_1186"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_216"
  op: "Sqrt"
  input: "add_742"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_743/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_743"
  op: "AddV2"
  input: "Sqrt_216"
  input: "add_743/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_219"
  op: "RealDiv"
  input: "add_741"
  input: "add_743"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_945"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1187/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1187"
  op: "Mul"
  input: "mul_1187/x"
  input: "ReadVariableOp_945"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_744"
  op: "AddV2"
  input: "truediv_219"
  input: "mul_1187"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1188"
  op: "Mul"
  input: "mul_21"
  input: "add_744"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_946"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_220"
  op: "Sub"
  input: "ReadVariableOp_946"
  input: "mul_1188"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_648"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_220"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_947"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_648"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_649"
  op: "AssignVariableOp"
  input: "Mul_1183/ReadVariableOp/Enter"
  input: "add_741"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_948"
  op: "ReadVariableOp"
  input: "Mul_1183/ReadVariableOp/Enter"
  input: "^AssignVariableOp_649"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_650"
  op: "AssignVariableOp"
  input: "Mul_1185/ReadVariableOp/Enter"
  input: "add_742"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_949"
  op: "ReadVariableOp"
  input: "Mul_1185/ReadVariableOp/Enter"
  input: "^AssignVariableOp_650"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1189/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1189/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1189/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1189/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1189"
  op: "Mul"
  input: "Mul_1189/x"
  input: "Mul_1189/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1190/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1190"
  op: "Mul"
  input: "Mul_1190/x"
  input: "CrossReplicaSum_217"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_745"
  op: "AddV2"
  input: "Mul_1189"
  input: "Mul_1190"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1191/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1191/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1191/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1191/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1191"
  op: "Mul"
  input: "Mul_1191/x"
  input: "Mul_1191/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_217"
  op: "Square"
  input: "CrossReplicaSum_217"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1192/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1192"
  op: "Mul"
  input: "Mul_1192/x"
  input: "Square_217"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_746"
  op: "AddV2"
  input: "Mul_1191"
  input: "Mul_1192"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_217"
  op: "Sqrt"
  input: "add_746"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_747/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_747"
  op: "AddV2"
  input: "Sqrt_217"
  input: "add_747/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_220"
  op: "RealDiv"
  input: "add_745"
  input: "add_747"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1193"
  op: "Mul"
  input: "mul_21"
  input: "truediv_220"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_950"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_221"
  op: "Sub"
  input: "ReadVariableOp_950"
  input: "mul_1193"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_651"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_221"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_951"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_651"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_652"
  op: "AssignVariableOp"
  input: "Mul_1189/ReadVariableOp/Enter"
  input: "add_745"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_952"
  op: "ReadVariableOp"
  input: "Mul_1189/ReadVariableOp/Enter"
  input: "^AssignVariableOp_652"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_653"
  op: "AssignVariableOp"
  input: "Mul_1191/ReadVariableOp/Enter"
  input: "add_746"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_953"
  op: "ReadVariableOp"
  input: "Mul_1191/ReadVariableOp/Enter"
  input: "^AssignVariableOp_653"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1194/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1194/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1194/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1194/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1194"
  op: "Mul"
  input: "Mul_1194/x"
  input: "Mul_1194/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1195/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1195"
  op: "Mul"
  input: "Mul_1195/x"
  input: "CrossReplicaSum_218"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_748"
  op: "AddV2"
  input: "Mul_1194"
  input: "Mul_1195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1196/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1196/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1196/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1196/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1196"
  op: "Mul"
  input: "Mul_1196/x"
  input: "Mul_1196/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_218"
  op: "Square"
  input: "CrossReplicaSum_218"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1197/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1197"
  op: "Mul"
  input: "Mul_1197/x"
  input: "Square_218"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_749"
  op: "AddV2"
  input: "Mul_1196"
  input: "Mul_1197"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_218"
  op: "Sqrt"
  input: "add_749"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_750/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_750"
  op: "AddV2"
  input: "Sqrt_218"
  input: "add_750/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_221"
  op: "RealDiv"
  input: "add_748"
  input: "add_750"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_954"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1198/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1198"
  op: "Mul"
  input: "mul_1198/x"
  input: "ReadVariableOp_954"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_751"
  op: "AddV2"
  input: "truediv_221"
  input: "mul_1198"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1199"
  op: "Mul"
  input: "mul_21"
  input: "add_751"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_955"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_222"
  op: "Sub"
  input: "ReadVariableOp_955"
  input: "mul_1199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_654"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_222"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_956"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_654"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_655"
  op: "AssignVariableOp"
  input: "Mul_1194/ReadVariableOp/Enter"
  input: "add_748"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_957"
  op: "ReadVariableOp"
  input: "Mul_1194/ReadVariableOp/Enter"
  input: "^AssignVariableOp_655"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_656"
  op: "AssignVariableOp"
  input: "Mul_1196/ReadVariableOp/Enter"
  input: "add_749"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_958"
  op: "ReadVariableOp"
  input: "Mul_1196/ReadVariableOp/Enter"
  input: "^AssignVariableOp_656"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_m"
  input: "electra/encoder/layer_0/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_v"
  input: "electra/encoder/layer_0/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1200/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1200/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1200/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1200/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1200"
  op: "Mul"
  input: "Mul_1200/x"
  input: "Mul_1200/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1201/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1201"
  op: "Mul"
  input: "Mul_1201/x"
  input: "CrossReplicaSum_219"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_752"
  op: "AddV2"
  input: "Mul_1200"
  input: "Mul_1201"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1202/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1202/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1202/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1202/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1202"
  op: "Mul"
  input: "Mul_1202/x"
  input: "Mul_1202/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_219"
  op: "Square"
  input: "CrossReplicaSum_219"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1203/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1203"
  op: "Mul"
  input: "Mul_1203/x"
  input: "Square_219"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_753"
  op: "AddV2"
  input: "Mul_1202"
  input: "Mul_1203"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_219"
  op: "Sqrt"
  input: "add_753"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_754/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_754"
  op: "AddV2"
  input: "Sqrt_219"
  input: "add_754/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_222"
  op: "RealDiv"
  input: "add_752"
  input: "add_754"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1204"
  op: "Mul"
  input: "mul_21"
  input: "truediv_222"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_959"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_223"
  op: "Sub"
  input: "ReadVariableOp_959"
  input: "mul_1204"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_657"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_223"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_960"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_657"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_658"
  op: "AssignVariableOp"
  input: "Mul_1200/ReadVariableOp/Enter"
  input: "add_752"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_961"
  op: "ReadVariableOp"
  input: "Mul_1200/ReadVariableOp/Enter"
  input: "^AssignVariableOp_658"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_659"
  op: "AssignVariableOp"
  input: "Mul_1202/ReadVariableOp/Enter"
  input: "add_753"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_962"
  op: "ReadVariableOp"
  input: "Mul_1202/ReadVariableOp/Enter"
  input: "^AssignVariableOp_659"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1205/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1205/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1205/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1205/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1205"
  op: "Mul"
  input: "Mul_1205/x"
  input: "Mul_1205/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1206/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1206"
  op: "Mul"
  input: "Mul_1206/x"
  input: "CrossReplicaSum_220"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_755"
  op: "AddV2"
  input: "Mul_1205"
  input: "Mul_1206"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1207/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1207/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1207/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1207/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1207"
  op: "Mul"
  input: "Mul_1207/x"
  input: "Mul_1207/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_220"
  op: "Square"
  input: "CrossReplicaSum_220"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1208/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1208"
  op: "Mul"
  input: "Mul_1208/x"
  input: "Square_220"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_756"
  op: "AddV2"
  input: "Mul_1207"
  input: "Mul_1208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_220"
  op: "Sqrt"
  input: "add_756"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_757/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_757"
  op: "AddV2"
  input: "Sqrt_220"
  input: "add_757/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_223"
  op: "RealDiv"
  input: "add_755"
  input: "add_757"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1209"
  op: "Mul"
  input: "mul_21"
  input: "truediv_223"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_963"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_224"
  op: "Sub"
  input: "ReadVariableOp_963"
  input: "mul_1209"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_660"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_224"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_964"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_660"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_661"
  op: "AssignVariableOp"
  input: "Mul_1205/ReadVariableOp/Enter"
  input: "add_755"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_965"
  op: "ReadVariableOp"
  input: "Mul_1205/ReadVariableOp/Enter"
  input: "^AssignVariableOp_661"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_662"
  op: "AssignVariableOp"
  input: "Mul_1207/ReadVariableOp/Enter"
  input: "add_756"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_966"
  op: "ReadVariableOp"
  input: "Mul_1207/ReadVariableOp/Enter"
  input: "^AssignVariableOp_662"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1210/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1210/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1210/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1210/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1210"
  op: "Mul"
  input: "Mul_1210/x"
  input: "Mul_1210/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1211/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1211"
  op: "Mul"
  input: "Mul_1211/x"
  input: "CrossReplicaSum_221"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_758"
  op: "AddV2"
  input: "Mul_1210"
  input: "Mul_1211"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1212/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1212/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1212/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1212/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1212"
  op: "Mul"
  input: "Mul_1212/x"
  input: "Mul_1212/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_221"
  op: "Square"
  input: "CrossReplicaSum_221"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1213/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1213"
  op: "Mul"
  input: "Mul_1213/x"
  input: "Square_221"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_759"
  op: "AddV2"
  input: "Mul_1212"
  input: "Mul_1213"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_221"
  op: "Sqrt"
  input: "add_759"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_760/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_760"
  op: "AddV2"
  input: "Sqrt_221"
  input: "add_760/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_224"
  op: "RealDiv"
  input: "add_758"
  input: "add_760"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1214"
  op: "Mul"
  input: "mul_21"
  input: "truediv_224"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_967"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_225"
  op: "Sub"
  input: "ReadVariableOp_967"
  input: "mul_1214"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_663"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_225"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_968"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_663"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_664"
  op: "AssignVariableOp"
  input: "Mul_1210/ReadVariableOp/Enter"
  input: "add_758"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_969"
  op: "ReadVariableOp"
  input: "Mul_1210/ReadVariableOp/Enter"
  input: "^AssignVariableOp_664"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_665"
  op: "AssignVariableOp"
  input: "Mul_1212/ReadVariableOp/Enter"
  input: "add_759"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_970"
  op: "ReadVariableOp"
  input: "Mul_1212/ReadVariableOp/Enter"
  input: "^AssignVariableOp_665"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1215/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1215/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1215/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1215/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1215"
  op: "Mul"
  input: "Mul_1215/x"
  input: "Mul_1215/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1216/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1216"
  op: "Mul"
  input: "Mul_1216/x"
  input: "CrossReplicaSum_222"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_761"
  op: "AddV2"
  input: "Mul_1215"
  input: "Mul_1216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1217/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1217/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1217/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1217/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1217"
  op: "Mul"
  input: "Mul_1217/x"
  input: "Mul_1217/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_222"
  op: "Square"
  input: "CrossReplicaSum_222"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1218/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1218"
  op: "Mul"
  input: "Mul_1218/x"
  input: "Square_222"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_762"
  op: "AddV2"
  input: "Mul_1217"
  input: "Mul_1218"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_222"
  op: "Sqrt"
  input: "add_762"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_763/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_763"
  op: "AddV2"
  input: "Sqrt_222"
  input: "add_763/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_225"
  op: "RealDiv"
  input: "add_761"
  input: "add_763"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_971"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1219/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1219"
  op: "Mul"
  input: "mul_1219/x"
  input: "ReadVariableOp_971"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_764"
  op: "AddV2"
  input: "truediv_225"
  input: "mul_1219"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1220"
  op: "Mul"
  input: "mul_21"
  input: "add_764"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_972"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_226"
  op: "Sub"
  input: "ReadVariableOp_972"
  input: "mul_1220"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_666"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_226"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_973"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_666"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_667"
  op: "AssignVariableOp"
  input: "Mul_1215/ReadVariableOp/Enter"
  input: "add_761"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_974"
  op: "ReadVariableOp"
  input: "Mul_1215/ReadVariableOp/Enter"
  input: "^AssignVariableOp_667"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_668"
  op: "AssignVariableOp"
  input: "Mul_1217/ReadVariableOp/Enter"
  input: "add_762"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_975"
  op: "ReadVariableOp"
  input: "Mul_1217/ReadVariableOp/Enter"
  input: "^AssignVariableOp_668"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1221/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1221/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1221/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1221/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1221"
  op: "Mul"
  input: "Mul_1221/x"
  input: "Mul_1221/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1222/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1222"
  op: "Mul"
  input: "Mul_1222/x"
  input: "CrossReplicaSum_223"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_765"
  op: "AddV2"
  input: "Mul_1221"
  input: "Mul_1222"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1223/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1223/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1223/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1223/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1223"
  op: "Mul"
  input: "Mul_1223/x"
  input: "Mul_1223/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_223"
  op: "Square"
  input: "CrossReplicaSum_223"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1224/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1224"
  op: "Mul"
  input: "Mul_1224/x"
  input: "Square_223"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_766"
  op: "AddV2"
  input: "Mul_1223"
  input: "Mul_1224"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_223"
  op: "Sqrt"
  input: "add_766"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_767/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_767"
  op: "AddV2"
  input: "Sqrt_223"
  input: "add_767/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_226"
  op: "RealDiv"
  input: "add_765"
  input: "add_767"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1225"
  op: "Mul"
  input: "mul_21"
  input: "truediv_226"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_976"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_227"
  op: "Sub"
  input: "ReadVariableOp_976"
  input: "mul_1225"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_669"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_227"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_977"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_669"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_670"
  op: "AssignVariableOp"
  input: "Mul_1221/ReadVariableOp/Enter"
  input: "add_765"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_978"
  op: "ReadVariableOp"
  input: "Mul_1221/ReadVariableOp/Enter"
  input: "^AssignVariableOp_670"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_671"
  op: "AssignVariableOp"
  input: "Mul_1223/ReadVariableOp/Enter"
  input: "add_766"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_979"
  op: "ReadVariableOp"
  input: "Mul_1223/ReadVariableOp/Enter"
  input: "^AssignVariableOp_671"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1226/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1226/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1226/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1226/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1226"
  op: "Mul"
  input: "Mul_1226/x"
  input: "Mul_1226/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1227/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1227"
  op: "Mul"
  input: "Mul_1227/x"
  input: "CrossReplicaSum_224"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_768"
  op: "AddV2"
  input: "Mul_1226"
  input: "Mul_1227"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1228/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1228/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1228/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1228/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1228"
  op: "Mul"
  input: "Mul_1228/x"
  input: "Mul_1228/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_224"
  op: "Square"
  input: "CrossReplicaSum_224"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1229/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1229"
  op: "Mul"
  input: "Mul_1229/x"
  input: "Square_224"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_769"
  op: "AddV2"
  input: "Mul_1228"
  input: "Mul_1229"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_224"
  op: "Sqrt"
  input: "add_769"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_770/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_770"
  op: "AddV2"
  input: "Sqrt_224"
  input: "add_770/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_227"
  op: "RealDiv"
  input: "add_768"
  input: "add_770"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_980"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1230/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1230"
  op: "Mul"
  input: "mul_1230/x"
  input: "ReadVariableOp_980"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_771"
  op: "AddV2"
  input: "truediv_227"
  input: "mul_1230"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1231"
  op: "Mul"
  input: "mul_21"
  input: "add_771"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_981"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_228"
  op: "Sub"
  input: "ReadVariableOp_981"
  input: "mul_1231"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_672"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_228"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_982"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_672"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_673"
  op: "AssignVariableOp"
  input: "Mul_1226/ReadVariableOp/Enter"
  input: "add_768"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_983"
  op: "ReadVariableOp"
  input: "Mul_1226/ReadVariableOp/Enter"
  input: "^AssignVariableOp_673"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_674"
  op: "AssignVariableOp"
  input: "Mul_1228/ReadVariableOp/Enter"
  input: "add_769"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_984"
  op: "ReadVariableOp"
  input: "Mul_1228/ReadVariableOp/Enter"
  input: "^AssignVariableOp_674"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1232/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1232/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1232/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1232/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1232"
  op: "Mul"
  input: "Mul_1232/x"
  input: "Mul_1232/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1233/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1233"
  op: "Mul"
  input: "Mul_1233/x"
  input: "CrossReplicaSum_225"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_772"
  op: "AddV2"
  input: "Mul_1232"
  input: "Mul_1233"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1234/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1234/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1234/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1234/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1234"
  op: "Mul"
  input: "Mul_1234/x"
  input: "Mul_1234/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_225"
  op: "Square"
  input: "CrossReplicaSum_225"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1235/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1235"
  op: "Mul"
  input: "Mul_1235/x"
  input: "Square_225"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_773"
  op: "AddV2"
  input: "Mul_1234"
  input: "Mul_1235"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_225"
  op: "Sqrt"
  input: "add_773"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_774/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_774"
  op: "AddV2"
  input: "Sqrt_225"
  input: "add_774/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_228"
  op: "RealDiv"
  input: "add_772"
  input: "add_774"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1236"
  op: "Mul"
  input: "mul_21"
  input: "truediv_228"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_985"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_229"
  op: "Sub"
  input: "ReadVariableOp_985"
  input: "mul_1236"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_675"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_229"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_986"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_675"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_676"
  op: "AssignVariableOp"
  input: "Mul_1232/ReadVariableOp/Enter"
  input: "add_772"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_987"
  op: "ReadVariableOp"
  input: "Mul_1232/ReadVariableOp/Enter"
  input: "^AssignVariableOp_676"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_677"
  op: "AssignVariableOp"
  input: "Mul_1234/ReadVariableOp/Enter"
  input: "add_773"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_988"
  op: "ReadVariableOp"
  input: "Mul_1234/ReadVariableOp/Enter"
  input: "^AssignVariableOp_677"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1237/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1237/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1237/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1237/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1237"
  op: "Mul"
  input: "Mul_1237/x"
  input: "Mul_1237/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1238/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1238"
  op: "Mul"
  input: "Mul_1238/x"
  input: "CrossReplicaSum_226"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_775"
  op: "AddV2"
  input: "Mul_1237"
  input: "Mul_1238"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1239/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1239/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1239/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1239/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1239"
  op: "Mul"
  input: "Mul_1239/x"
  input: "Mul_1239/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_226"
  op: "Square"
  input: "CrossReplicaSum_226"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1240/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1240"
  op: "Mul"
  input: "Mul_1240/x"
  input: "Square_226"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_776"
  op: "AddV2"
  input: "Mul_1239"
  input: "Mul_1240"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_226"
  op: "Sqrt"
  input: "add_776"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_777/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_777"
  op: "AddV2"
  input: "Sqrt_226"
  input: "add_777/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_229"
  op: "RealDiv"
  input: "add_775"
  input: "add_777"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_989"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1241/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1241"
  op: "Mul"
  input: "mul_1241/x"
  input: "ReadVariableOp_989"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_778"
  op: "AddV2"
  input: "truediv_229"
  input: "mul_1241"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1242"
  op: "Mul"
  input: "mul_21"
  input: "add_778"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_990"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_230"
  op: "Sub"
  input: "ReadVariableOp_990"
  input: "mul_1242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_678"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_230"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_991"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_678"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_679"
  op: "AssignVariableOp"
  input: "Mul_1237/ReadVariableOp/Enter"
  input: "add_775"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_992"
  op: "ReadVariableOp"
  input: "Mul_1237/ReadVariableOp/Enter"
  input: "^AssignVariableOp_679"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_680"
  op: "AssignVariableOp"
  input: "Mul_1239/ReadVariableOp/Enter"
  input: "add_776"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_993"
  op: "ReadVariableOp"
  input: "Mul_1239/ReadVariableOp/Enter"
  input: "^AssignVariableOp_680"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1243/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1243/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1243/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1243/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1243"
  op: "Mul"
  input: "Mul_1243/x"
  input: "Mul_1243/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1244/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1244"
  op: "Mul"
  input: "Mul_1244/x"
  input: "CrossReplicaSum_227"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_779"
  op: "AddV2"
  input: "Mul_1243"
  input: "Mul_1244"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1245/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1245/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1245/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1245/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1245"
  op: "Mul"
  input: "Mul_1245/x"
  input: "Mul_1245/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_227"
  op: "Square"
  input: "CrossReplicaSum_227"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1246/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1246"
  op: "Mul"
  input: "Mul_1246/x"
  input: "Square_227"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_780"
  op: "AddV2"
  input: "Mul_1245"
  input: "Mul_1246"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_227"
  op: "Sqrt"
  input: "add_780"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_781/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_781"
  op: "AddV2"
  input: "Sqrt_227"
  input: "add_781/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_230"
  op: "RealDiv"
  input: "add_779"
  input: "add_781"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1247"
  op: "Mul"
  input: "mul_21"
  input: "truediv_230"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_994"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_231"
  op: "Sub"
  input: "ReadVariableOp_994"
  input: "mul_1247"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_681"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_231"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_995"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_681"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_682"
  op: "AssignVariableOp"
  input: "Mul_1243/ReadVariableOp/Enter"
  input: "add_779"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_996"
  op: "ReadVariableOp"
  input: "Mul_1243/ReadVariableOp/Enter"
  input: "^AssignVariableOp_682"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_683"
  op: "AssignVariableOp"
  input: "Mul_1245/ReadVariableOp/Enter"
  input: "add_780"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_997"
  op: "ReadVariableOp"
  input: "Mul_1245/ReadVariableOp/Enter"
  input: "^AssignVariableOp_683"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1248/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1248/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1248/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1248/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1248"
  op: "Mul"
  input: "Mul_1248/x"
  input: "Mul_1248/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1249/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1249"
  op: "Mul"
  input: "Mul_1249/x"
  input: "CrossReplicaSum_228"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_782"
  op: "AddV2"
  input: "Mul_1248"
  input: "Mul_1249"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1250/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1250/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1250/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1250/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1250"
  op: "Mul"
  input: "Mul_1250/x"
  input: "Mul_1250/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_228"
  op: "Square"
  input: "CrossReplicaSum_228"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1251/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1251"
  op: "Mul"
  input: "Mul_1251/x"
  input: "Square_228"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_783"
  op: "AddV2"
  input: "Mul_1250"
  input: "Mul_1251"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_228"
  op: "Sqrt"
  input: "add_783"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_784/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_784"
  op: "AddV2"
  input: "Sqrt_228"
  input: "add_784/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_231"
  op: "RealDiv"
  input: "add_782"
  input: "add_784"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_998"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1252/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1252"
  op: "Mul"
  input: "mul_1252/x"
  input: "ReadVariableOp_998"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_785"
  op: "AddV2"
  input: "truediv_231"
  input: "mul_1252"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1253"
  op: "Mul"
  input: "mul_21"
  input: "add_785"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_999"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_232"
  op: "Sub"
  input: "ReadVariableOp_999"
  input: "mul_1253"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_684"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_232"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1000"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_684"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_685"
  op: "AssignVariableOp"
  input: "Mul_1248/ReadVariableOp/Enter"
  input: "add_782"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1001"
  op: "ReadVariableOp"
  input: "Mul_1248/ReadVariableOp/Enter"
  input: "^AssignVariableOp_685"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_686"
  op: "AssignVariableOp"
  input: "Mul_1250/ReadVariableOp/Enter"
  input: "add_783"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1002"
  op: "ReadVariableOp"
  input: "Mul_1250/ReadVariableOp/Enter"
  input: "^AssignVariableOp_686"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1254/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1254/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1254/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1254/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1254"
  op: "Mul"
  input: "Mul_1254/x"
  input: "Mul_1254/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1255/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1255"
  op: "Mul"
  input: "Mul_1255/x"
  input: "CrossReplicaSum_229"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_786"
  op: "AddV2"
  input: "Mul_1254"
  input: "Mul_1255"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1256/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1256/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1256/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1256/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1256"
  op: "Mul"
  input: "Mul_1256/x"
  input: "Mul_1256/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_229"
  op: "Square"
  input: "CrossReplicaSum_229"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1257/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1257"
  op: "Mul"
  input: "Mul_1257/x"
  input: "Square_229"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_787"
  op: "AddV2"
  input: "Mul_1256"
  input: "Mul_1257"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_229"
  op: "Sqrt"
  input: "add_787"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_788/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_788"
  op: "AddV2"
  input: "Sqrt_229"
  input: "add_788/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_232"
  op: "RealDiv"
  input: "add_786"
  input: "add_788"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1258"
  op: "Mul"
  input: "mul_21"
  input: "truediv_232"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1003"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_233"
  op: "Sub"
  input: "ReadVariableOp_1003"
  input: "mul_1258"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_687"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_233"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1004"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_687"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_688"
  op: "AssignVariableOp"
  input: "Mul_1254/ReadVariableOp/Enter"
  input: "add_786"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1005"
  op: "ReadVariableOp"
  input: "Mul_1254/ReadVariableOp/Enter"
  input: "^AssignVariableOp_688"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_689"
  op: "AssignVariableOp"
  input: "Mul_1256/ReadVariableOp/Enter"
  input: "add_787"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1006"
  op: "ReadVariableOp"
  input: "Mul_1256/ReadVariableOp/Enter"
  input: "^AssignVariableOp_689"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1259/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1259/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1259/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1259/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1259"
  op: "Mul"
  input: "Mul_1259/x"
  input: "Mul_1259/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1260/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1260"
  op: "Mul"
  input: "Mul_1260/x"
  input: "CrossReplicaSum_230"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_789"
  op: "AddV2"
  input: "Mul_1259"
  input: "Mul_1260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1261/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1261/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1261/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1261/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1261"
  op: "Mul"
  input: "Mul_1261/x"
  input: "Mul_1261/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_230"
  op: "Square"
  input: "CrossReplicaSum_230"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1262/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1262"
  op: "Mul"
  input: "Mul_1262/x"
  input: "Square_230"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_790"
  op: "AddV2"
  input: "Mul_1261"
  input: "Mul_1262"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_230"
  op: "Sqrt"
  input: "add_790"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_791/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_791"
  op: "AddV2"
  input: "Sqrt_230"
  input: "add_791/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_233"
  op: "RealDiv"
  input: "add_789"
  input: "add_791"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1263"
  op: "Mul"
  input: "mul_21"
  input: "truediv_233"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1007"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_234"
  op: "Sub"
  input: "ReadVariableOp_1007"
  input: "mul_1263"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_690"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_234"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1008"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_690"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_691"
  op: "AssignVariableOp"
  input: "Mul_1259/ReadVariableOp/Enter"
  input: "add_789"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1009"
  op: "ReadVariableOp"
  input: "Mul_1259/ReadVariableOp/Enter"
  input: "^AssignVariableOp_691"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_692"
  op: "AssignVariableOp"
  input: "Mul_1261/ReadVariableOp/Enter"
  input: "add_790"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1010"
  op: "ReadVariableOp"
  input: "Mul_1261/ReadVariableOp/Enter"
  input: "^AssignVariableOp_692"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1264/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1264/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1264/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1264/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1264"
  op: "Mul"
  input: "Mul_1264/x"
  input: "Mul_1264/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1265/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1265"
  op: "Mul"
  input: "Mul_1265/x"
  input: "CrossReplicaSum_231"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_792"
  op: "AddV2"
  input: "Mul_1264"
  input: "Mul_1265"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1266/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1266/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1266/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1266/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1266"
  op: "Mul"
  input: "Mul_1266/x"
  input: "Mul_1266/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_231"
  op: "Square"
  input: "CrossReplicaSum_231"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1267/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1267"
  op: "Mul"
  input: "Mul_1267/x"
  input: "Square_231"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_793"
  op: "AddV2"
  input: "Mul_1266"
  input: "Mul_1267"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_231"
  op: "Sqrt"
  input: "add_793"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_794/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_794"
  op: "AddV2"
  input: "Sqrt_231"
  input: "add_794/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_234"
  op: "RealDiv"
  input: "add_792"
  input: "add_794"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1268"
  op: "Mul"
  input: "mul_21"
  input: "truediv_234"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1011"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_235"
  op: "Sub"
  input: "ReadVariableOp_1011"
  input: "mul_1268"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_693"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_235"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1012"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_693"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_694"
  op: "AssignVariableOp"
  input: "Mul_1264/ReadVariableOp/Enter"
  input: "add_792"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1013"
  op: "ReadVariableOp"
  input: "Mul_1264/ReadVariableOp/Enter"
  input: "^AssignVariableOp_694"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_695"
  op: "AssignVariableOp"
  input: "Mul_1266/ReadVariableOp/Enter"
  input: "add_793"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1014"
  op: "ReadVariableOp"
  input: "Mul_1266/ReadVariableOp/Enter"
  input: "^AssignVariableOp_695"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1269/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1269/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1269/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1269/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1269"
  op: "Mul"
  input: "Mul_1269/x"
  input: "Mul_1269/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1270/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1270"
  op: "Mul"
  input: "Mul_1270/x"
  input: "CrossReplicaSum_232"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_795"
  op: "AddV2"
  input: "Mul_1269"
  input: "Mul_1270"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1271/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1271/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1271/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1271/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1271"
  op: "Mul"
  input: "Mul_1271/x"
  input: "Mul_1271/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_232"
  op: "Square"
  input: "CrossReplicaSum_232"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1272/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1272"
  op: "Mul"
  input: "Mul_1272/x"
  input: "Square_232"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_796"
  op: "AddV2"
  input: "Mul_1271"
  input: "Mul_1272"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_232"
  op: "Sqrt"
  input: "add_796"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_797/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_797"
  op: "AddV2"
  input: "Sqrt_232"
  input: "add_797/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_235"
  op: "RealDiv"
  input: "add_795"
  input: "add_797"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1015"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1273/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1273"
  op: "Mul"
  input: "mul_1273/x"
  input: "ReadVariableOp_1015"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_798"
  op: "AddV2"
  input: "truediv_235"
  input: "mul_1273"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1274"
  op: "Mul"
  input: "mul_21"
  input: "add_798"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1016"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_236"
  op: "Sub"
  input: "ReadVariableOp_1016"
  input: "mul_1274"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_696"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_236"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1017"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_696"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_697"
  op: "AssignVariableOp"
  input: "Mul_1269/ReadVariableOp/Enter"
  input: "add_795"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1018"
  op: "ReadVariableOp"
  input: "Mul_1269/ReadVariableOp/Enter"
  input: "^AssignVariableOp_697"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_698"
  op: "AssignVariableOp"
  input: "Mul_1271/ReadVariableOp/Enter"
  input: "add_796"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1019"
  op: "ReadVariableOp"
  input: "Mul_1271/ReadVariableOp/Enter"
  input: "^AssignVariableOp_698"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1275/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1275/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1275/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1275/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1275"
  op: "Mul"
  input: "Mul_1275/x"
  input: "Mul_1275/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1276/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1276"
  op: "Mul"
  input: "Mul_1276/x"
  input: "CrossReplicaSum_233"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_799"
  op: "AddV2"
  input: "Mul_1275"
  input: "Mul_1276"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1277/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1277/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1277/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1277/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1277"
  op: "Mul"
  input: "Mul_1277/x"
  input: "Mul_1277/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_233"
  op: "Square"
  input: "CrossReplicaSum_233"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1278/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1278"
  op: "Mul"
  input: "Mul_1278/x"
  input: "Square_233"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_800"
  op: "AddV2"
  input: "Mul_1277"
  input: "Mul_1278"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_233"
  op: "Sqrt"
  input: "add_800"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_801/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_801"
  op: "AddV2"
  input: "Sqrt_233"
  input: "add_801/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_236"
  op: "RealDiv"
  input: "add_799"
  input: "add_801"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1279"
  op: "Mul"
  input: "mul_21"
  input: "truediv_236"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1020"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_237"
  op: "Sub"
  input: "ReadVariableOp_1020"
  input: "mul_1279"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_699"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_237"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1021"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_699"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_700"
  op: "AssignVariableOp"
  input: "Mul_1275/ReadVariableOp/Enter"
  input: "add_799"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1022"
  op: "ReadVariableOp"
  input: "Mul_1275/ReadVariableOp/Enter"
  input: "^AssignVariableOp_700"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_701"
  op: "AssignVariableOp"
  input: "Mul_1277/ReadVariableOp/Enter"
  input: "add_800"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1023"
  op: "ReadVariableOp"
  input: "Mul_1277/ReadVariableOp/Enter"
  input: "^AssignVariableOp_701"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1280/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1280/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1280/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1280/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1280"
  op: "Mul"
  input: "Mul_1280/x"
  input: "Mul_1280/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1281/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1281"
  op: "Mul"
  input: "Mul_1281/x"
  input: "CrossReplicaSum_234"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_802"
  op: "AddV2"
  input: "Mul_1280"
  input: "Mul_1281"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1282/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1282/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1282/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1282/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1282"
  op: "Mul"
  input: "Mul_1282/x"
  input: "Mul_1282/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_234"
  op: "Square"
  input: "CrossReplicaSum_234"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1283/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1283"
  op: "Mul"
  input: "Mul_1283/x"
  input: "Square_234"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_803"
  op: "AddV2"
  input: "Mul_1282"
  input: "Mul_1283"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_234"
  op: "Sqrt"
  input: "add_803"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_804/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_804"
  op: "AddV2"
  input: "Sqrt_234"
  input: "add_804/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_237"
  op: "RealDiv"
  input: "add_802"
  input: "add_804"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1024"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1284/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1284"
  op: "Mul"
  input: "mul_1284/x"
  input: "ReadVariableOp_1024"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_805"
  op: "AddV2"
  input: "truediv_237"
  input: "mul_1284"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1285"
  op: "Mul"
  input: "mul_21"
  input: "add_805"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1025"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_238"
  op: "Sub"
  input: "ReadVariableOp_1025"
  input: "mul_1285"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_702"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_238"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1026"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_702"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_703"
  op: "AssignVariableOp"
  input: "Mul_1280/ReadVariableOp/Enter"
  input: "add_802"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1027"
  op: "ReadVariableOp"
  input: "Mul_1280/ReadVariableOp/Enter"
  input: "^AssignVariableOp_703"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_704"
  op: "AssignVariableOp"
  input: "Mul_1282/ReadVariableOp/Enter"
  input: "add_803"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1028"
  op: "ReadVariableOp"
  input: "Mul_1282/ReadVariableOp/Enter"
  input: "^AssignVariableOp_704"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_m"
  input: "electra/encoder/layer_1/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_v"
  input: "electra/encoder/layer_1/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1286/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1286/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1286/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1286/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1286"
  op: "Mul"
  input: "Mul_1286/x"
  input: "Mul_1286/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1287/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1287"
  op: "Mul"
  input: "Mul_1287/x"
  input: "CrossReplicaSum_235"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_806"
  op: "AddV2"
  input: "Mul_1286"
  input: "Mul_1287"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1288/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1288/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1288/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1288/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1288"
  op: "Mul"
  input: "Mul_1288/x"
  input: "Mul_1288/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_235"
  op: "Square"
  input: "CrossReplicaSum_235"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1289/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1289"
  op: "Mul"
  input: "Mul_1289/x"
  input: "Square_235"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_807"
  op: "AddV2"
  input: "Mul_1288"
  input: "Mul_1289"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_235"
  op: "Sqrt"
  input: "add_807"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_808/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_808"
  op: "AddV2"
  input: "Sqrt_235"
  input: "add_808/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_238"
  op: "RealDiv"
  input: "add_806"
  input: "add_808"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1290"
  op: "Mul"
  input: "mul_21"
  input: "truediv_238"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1029"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_239"
  op: "Sub"
  input: "ReadVariableOp_1029"
  input: "mul_1290"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_705"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_239"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1030"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_705"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_706"
  op: "AssignVariableOp"
  input: "Mul_1286/ReadVariableOp/Enter"
  input: "add_806"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1031"
  op: "ReadVariableOp"
  input: "Mul_1286/ReadVariableOp/Enter"
  input: "^AssignVariableOp_706"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_707"
  op: "AssignVariableOp"
  input: "Mul_1288/ReadVariableOp/Enter"
  input: "add_807"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1032"
  op: "ReadVariableOp"
  input: "Mul_1288/ReadVariableOp/Enter"
  input: "^AssignVariableOp_707"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1291/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1291/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1291/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1291/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1291"
  op: "Mul"
  input: "Mul_1291/x"
  input: "Mul_1291/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1292/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1292"
  op: "Mul"
  input: "Mul_1292/x"
  input: "CrossReplicaSum_236"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_809"
  op: "AddV2"
  input: "Mul_1291"
  input: "Mul_1292"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1293/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1293/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1293/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1293/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1293"
  op: "Mul"
  input: "Mul_1293/x"
  input: "Mul_1293/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_236"
  op: "Square"
  input: "CrossReplicaSum_236"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1294/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1294"
  op: "Mul"
  input: "Mul_1294/x"
  input: "Square_236"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_810"
  op: "AddV2"
  input: "Mul_1293"
  input: "Mul_1294"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_236"
  op: "Sqrt"
  input: "add_810"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_811/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_811"
  op: "AddV2"
  input: "Sqrt_236"
  input: "add_811/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_239"
  op: "RealDiv"
  input: "add_809"
  input: "add_811"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1295"
  op: "Mul"
  input: "mul_21"
  input: "truediv_239"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1033"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_240"
  op: "Sub"
  input: "ReadVariableOp_1033"
  input: "mul_1295"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_708"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_240"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1034"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_708"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_709"
  op: "AssignVariableOp"
  input: "Mul_1291/ReadVariableOp/Enter"
  input: "add_809"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1035"
  op: "ReadVariableOp"
  input: "Mul_1291/ReadVariableOp/Enter"
  input: "^AssignVariableOp_709"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_710"
  op: "AssignVariableOp"
  input: "Mul_1293/ReadVariableOp/Enter"
  input: "add_810"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1036"
  op: "ReadVariableOp"
  input: "Mul_1293/ReadVariableOp/Enter"
  input: "^AssignVariableOp_710"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1296/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1296/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1296/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1296/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1296"
  op: "Mul"
  input: "Mul_1296/x"
  input: "Mul_1296/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1297/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1297"
  op: "Mul"
  input: "Mul_1297/x"
  input: "CrossReplicaSum_237"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_812"
  op: "AddV2"
  input: "Mul_1296"
  input: "Mul_1297"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1298/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1298/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1298/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1298/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1298"
  op: "Mul"
  input: "Mul_1298/x"
  input: "Mul_1298/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_237"
  op: "Square"
  input: "CrossReplicaSum_237"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1299/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1299"
  op: "Mul"
  input: "Mul_1299/x"
  input: "Square_237"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_813"
  op: "AddV2"
  input: "Mul_1298"
  input: "Mul_1299"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_237"
  op: "Sqrt"
  input: "add_813"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_814/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_814"
  op: "AddV2"
  input: "Sqrt_237"
  input: "add_814/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_240"
  op: "RealDiv"
  input: "add_812"
  input: "add_814"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1300"
  op: "Mul"
  input: "mul_21"
  input: "truediv_240"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1037"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_241"
  op: "Sub"
  input: "ReadVariableOp_1037"
  input: "mul_1300"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_711"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_241"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1038"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_711"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_712"
  op: "AssignVariableOp"
  input: "Mul_1296/ReadVariableOp/Enter"
  input: "add_812"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1039"
  op: "ReadVariableOp"
  input: "Mul_1296/ReadVariableOp/Enter"
  input: "^AssignVariableOp_712"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_713"
  op: "AssignVariableOp"
  input: "Mul_1298/ReadVariableOp/Enter"
  input: "add_813"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1040"
  op: "ReadVariableOp"
  input: "Mul_1298/ReadVariableOp/Enter"
  input: "^AssignVariableOp_713"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1301/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1301/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1301/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1301/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1301"
  op: "Mul"
  input: "Mul_1301/x"
  input: "Mul_1301/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1302/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1302"
  op: "Mul"
  input: "Mul_1302/x"
  input: "CrossReplicaSum_238"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_815"
  op: "AddV2"
  input: "Mul_1301"
  input: "Mul_1302"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1303/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1303/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1303/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1303/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1303"
  op: "Mul"
  input: "Mul_1303/x"
  input: "Mul_1303/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_238"
  op: "Square"
  input: "CrossReplicaSum_238"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1304/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1304"
  op: "Mul"
  input: "Mul_1304/x"
  input: "Square_238"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_816"
  op: "AddV2"
  input: "Mul_1303"
  input: "Mul_1304"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_238"
  op: "Sqrt"
  input: "add_816"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_817/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_817"
  op: "AddV2"
  input: "Sqrt_238"
  input: "add_817/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_241"
  op: "RealDiv"
  input: "add_815"
  input: "add_817"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1041"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1305/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1305"
  op: "Mul"
  input: "mul_1305/x"
  input: "ReadVariableOp_1041"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_818"
  op: "AddV2"
  input: "truediv_241"
  input: "mul_1305"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1306"
  op: "Mul"
  input: "mul_21"
  input: "add_818"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1042"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_242"
  op: "Sub"
  input: "ReadVariableOp_1042"
  input: "mul_1306"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_714"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_242"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1043"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_714"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_715"
  op: "AssignVariableOp"
  input: "Mul_1301/ReadVariableOp/Enter"
  input: "add_815"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1044"
  op: "ReadVariableOp"
  input: "Mul_1301/ReadVariableOp/Enter"
  input: "^AssignVariableOp_715"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_716"
  op: "AssignVariableOp"
  input: "Mul_1303/ReadVariableOp/Enter"
  input: "add_816"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1045"
  op: "ReadVariableOp"
  input: "Mul_1303/ReadVariableOp/Enter"
  input: "^AssignVariableOp_716"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1307/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1307/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1307/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1307/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1307"
  op: "Mul"
  input: "Mul_1307/x"
  input: "Mul_1307/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1308/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1308"
  op: "Mul"
  input: "Mul_1308/x"
  input: "CrossReplicaSum_239"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_819"
  op: "AddV2"
  input: "Mul_1307"
  input: "Mul_1308"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1309/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1309/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1309/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1309/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1309"
  op: "Mul"
  input: "Mul_1309/x"
  input: "Mul_1309/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_239"
  op: "Square"
  input: "CrossReplicaSum_239"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1310/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1310"
  op: "Mul"
  input: "Mul_1310/x"
  input: "Square_239"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_820"
  op: "AddV2"
  input: "Mul_1309"
  input: "Mul_1310"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_239"
  op: "Sqrt"
  input: "add_820"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_821/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_821"
  op: "AddV2"
  input: "Sqrt_239"
  input: "add_821/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_242"
  op: "RealDiv"
  input: "add_819"
  input: "add_821"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1311"
  op: "Mul"
  input: "mul_21"
  input: "truediv_242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1046"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_243"
  op: "Sub"
  input: "ReadVariableOp_1046"
  input: "mul_1311"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_717"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_243"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1047"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_717"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_718"
  op: "AssignVariableOp"
  input: "Mul_1307/ReadVariableOp/Enter"
  input: "add_819"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1048"
  op: "ReadVariableOp"
  input: "Mul_1307/ReadVariableOp/Enter"
  input: "^AssignVariableOp_718"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_719"
  op: "AssignVariableOp"
  input: "Mul_1309/ReadVariableOp/Enter"
  input: "add_820"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1049"
  op: "ReadVariableOp"
  input: "Mul_1309/ReadVariableOp/Enter"
  input: "^AssignVariableOp_719"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1312/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1312/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1312/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1312/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1312"
  op: "Mul"
  input: "Mul_1312/x"
  input: "Mul_1312/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1313/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1313"
  op: "Mul"
  input: "Mul_1313/x"
  input: "CrossReplicaSum_240"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_822"
  op: "AddV2"
  input: "Mul_1312"
  input: "Mul_1313"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1314/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1314/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1314/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1314/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1314"
  op: "Mul"
  input: "Mul_1314/x"
  input: "Mul_1314/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_240"
  op: "Square"
  input: "CrossReplicaSum_240"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1315/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1315"
  op: "Mul"
  input: "Mul_1315/x"
  input: "Square_240"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_823"
  op: "AddV2"
  input: "Mul_1314"
  input: "Mul_1315"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_240"
  op: "Sqrt"
  input: "add_823"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_824/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_824"
  op: "AddV2"
  input: "Sqrt_240"
  input: "add_824/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_243"
  op: "RealDiv"
  input: "add_822"
  input: "add_824"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1050"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1316/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1316"
  op: "Mul"
  input: "mul_1316/x"
  input: "ReadVariableOp_1050"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_825"
  op: "AddV2"
  input: "truediv_243"
  input: "mul_1316"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1317"
  op: "Mul"
  input: "mul_21"
  input: "add_825"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1051"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_244"
  op: "Sub"
  input: "ReadVariableOp_1051"
  input: "mul_1317"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_720"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_244"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1052"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_720"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_721"
  op: "AssignVariableOp"
  input: "Mul_1312/ReadVariableOp/Enter"
  input: "add_822"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1053"
  op: "ReadVariableOp"
  input: "Mul_1312/ReadVariableOp/Enter"
  input: "^AssignVariableOp_721"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_722"
  op: "AssignVariableOp"
  input: "Mul_1314/ReadVariableOp/Enter"
  input: "add_823"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1054"
  op: "ReadVariableOp"
  input: "Mul_1314/ReadVariableOp/Enter"
  input: "^AssignVariableOp_722"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1318/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1318/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1318/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1318/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1318"
  op: "Mul"
  input: "Mul_1318/x"
  input: "Mul_1318/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1319/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1319"
  op: "Mul"
  input: "Mul_1319/x"
  input: "CrossReplicaSum_241"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_826"
  op: "AddV2"
  input: "Mul_1318"
  input: "Mul_1319"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1320/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1320/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1320/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1320/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1320"
  op: "Mul"
  input: "Mul_1320/x"
  input: "Mul_1320/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_241"
  op: "Square"
  input: "CrossReplicaSum_241"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1321/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1321"
  op: "Mul"
  input: "Mul_1321/x"
  input: "Square_241"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_827"
  op: "AddV2"
  input: "Mul_1320"
  input: "Mul_1321"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_241"
  op: "Sqrt"
  input: "add_827"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_828/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_828"
  op: "AddV2"
  input: "Sqrt_241"
  input: "add_828/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_244"
  op: "RealDiv"
  input: "add_826"
  input: "add_828"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1322"
  op: "Mul"
  input: "mul_21"
  input: "truediv_244"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1055"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_245"
  op: "Sub"
  input: "ReadVariableOp_1055"
  input: "mul_1322"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_723"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_245"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1056"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_723"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_724"
  op: "AssignVariableOp"
  input: "Mul_1318/ReadVariableOp/Enter"
  input: "add_826"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1057"
  op: "ReadVariableOp"
  input: "Mul_1318/ReadVariableOp/Enter"
  input: "^AssignVariableOp_724"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_725"
  op: "AssignVariableOp"
  input: "Mul_1320/ReadVariableOp/Enter"
  input: "add_827"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1058"
  op: "ReadVariableOp"
  input: "Mul_1320/ReadVariableOp/Enter"
  input: "^AssignVariableOp_725"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1323/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1323/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1323/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1323/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1323"
  op: "Mul"
  input: "Mul_1323/x"
  input: "Mul_1323/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1324/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1324"
  op: "Mul"
  input: "Mul_1324/x"
  input: "CrossReplicaSum_242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_829"
  op: "AddV2"
  input: "Mul_1323"
  input: "Mul_1324"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1325/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1325/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1325/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1325/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1325"
  op: "Mul"
  input: "Mul_1325/x"
  input: "Mul_1325/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_242"
  op: "Square"
  input: "CrossReplicaSum_242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1326/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1326"
  op: "Mul"
  input: "Mul_1326/x"
  input: "Square_242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_830"
  op: "AddV2"
  input: "Mul_1325"
  input: "Mul_1326"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_242"
  op: "Sqrt"
  input: "add_830"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_831/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_831"
  op: "AddV2"
  input: "Sqrt_242"
  input: "add_831/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_245"
  op: "RealDiv"
  input: "add_829"
  input: "add_831"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1059"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1327/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1327"
  op: "Mul"
  input: "mul_1327/x"
  input: "ReadVariableOp_1059"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_832"
  op: "AddV2"
  input: "truediv_245"
  input: "mul_1327"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1328"
  op: "Mul"
  input: "mul_21"
  input: "add_832"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1060"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_246"
  op: "Sub"
  input: "ReadVariableOp_1060"
  input: "mul_1328"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_726"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_246"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1061"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_726"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_727"
  op: "AssignVariableOp"
  input: "Mul_1323/ReadVariableOp/Enter"
  input: "add_829"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1062"
  op: "ReadVariableOp"
  input: "Mul_1323/ReadVariableOp/Enter"
  input: "^AssignVariableOp_727"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_728"
  op: "AssignVariableOp"
  input: "Mul_1325/ReadVariableOp/Enter"
  input: "add_830"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1063"
  op: "ReadVariableOp"
  input: "Mul_1325/ReadVariableOp/Enter"
  input: "^AssignVariableOp_728"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1329/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1329/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1329/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1329/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1329"
  op: "Mul"
  input: "Mul_1329/x"
  input: "Mul_1329/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1330/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1330"
  op: "Mul"
  input: "Mul_1330/x"
  input: "CrossReplicaSum_243"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_833"
  op: "AddV2"
  input: "Mul_1329"
  input: "Mul_1330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1331/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1331/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1331/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1331/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1331"
  op: "Mul"
  input: "Mul_1331/x"
  input: "Mul_1331/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_243"
  op: "Square"
  input: "CrossReplicaSum_243"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1332/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1332"
  op: "Mul"
  input: "Mul_1332/x"
  input: "Square_243"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_834"
  op: "AddV2"
  input: "Mul_1331"
  input: "Mul_1332"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_243"
  op: "Sqrt"
  input: "add_834"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_835/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_835"
  op: "AddV2"
  input: "Sqrt_243"
  input: "add_835/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_246"
  op: "RealDiv"
  input: "add_833"
  input: "add_835"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1333"
  op: "Mul"
  input: "mul_21"
  input: "truediv_246"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1064"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_247"
  op: "Sub"
  input: "ReadVariableOp_1064"
  input: "mul_1333"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_729"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_247"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1065"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_729"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_730"
  op: "AssignVariableOp"
  input: "Mul_1329/ReadVariableOp/Enter"
  input: "add_833"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1066"
  op: "ReadVariableOp"
  input: "Mul_1329/ReadVariableOp/Enter"
  input: "^AssignVariableOp_730"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_731"
  op: "AssignVariableOp"
  input: "Mul_1331/ReadVariableOp/Enter"
  input: "add_834"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1067"
  op: "ReadVariableOp"
  input: "Mul_1331/ReadVariableOp/Enter"
  input: "^AssignVariableOp_731"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1334/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1334/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1334/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1334/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1334"
  op: "Mul"
  input: "Mul_1334/x"
  input: "Mul_1334/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1335/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1335"
  op: "Mul"
  input: "Mul_1335/x"
  input: "CrossReplicaSum_244"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_836"
  op: "AddV2"
  input: "Mul_1334"
  input: "Mul_1335"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1336/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1336/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1336/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1336/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1336"
  op: "Mul"
  input: "Mul_1336/x"
  input: "Mul_1336/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_244"
  op: "Square"
  input: "CrossReplicaSum_244"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1337/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1337"
  op: "Mul"
  input: "Mul_1337/x"
  input: "Square_244"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_837"
  op: "AddV2"
  input: "Mul_1336"
  input: "Mul_1337"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_244"
  op: "Sqrt"
  input: "add_837"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_838/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_838"
  op: "AddV2"
  input: "Sqrt_244"
  input: "add_838/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_247"
  op: "RealDiv"
  input: "add_836"
  input: "add_838"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1068"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1338/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1338"
  op: "Mul"
  input: "mul_1338/x"
  input: "ReadVariableOp_1068"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_839"
  op: "AddV2"
  input: "truediv_247"
  input: "mul_1338"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1339"
  op: "Mul"
  input: "mul_21"
  input: "add_839"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1069"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_248"
  op: "Sub"
  input: "ReadVariableOp_1069"
  input: "mul_1339"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_732"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_248"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1070"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_732"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_733"
  op: "AssignVariableOp"
  input: "Mul_1334/ReadVariableOp/Enter"
  input: "add_836"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1071"
  op: "ReadVariableOp"
  input: "Mul_1334/ReadVariableOp/Enter"
  input: "^AssignVariableOp_733"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_734"
  op: "AssignVariableOp"
  input: "Mul_1336/ReadVariableOp/Enter"
  input: "add_837"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1072"
  op: "ReadVariableOp"
  input: "Mul_1336/ReadVariableOp/Enter"
  input: "^AssignVariableOp_734"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1340/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1340/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1340/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1340/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1340"
  op: "Mul"
  input: "Mul_1340/x"
  input: "Mul_1340/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1341/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1341"
  op: "Mul"
  input: "Mul_1341/x"
  input: "CrossReplicaSum_245"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_840"
  op: "AddV2"
  input: "Mul_1340"
  input: "Mul_1341"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1342/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1342/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1342/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1342/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1342"
  op: "Mul"
  input: "Mul_1342/x"
  input: "Mul_1342/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_245"
  op: "Square"
  input: "CrossReplicaSum_245"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1343/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1343"
  op: "Mul"
  input: "Mul_1343/x"
  input: "Square_245"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_841"
  op: "AddV2"
  input: "Mul_1342"
  input: "Mul_1343"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_245"
  op: "Sqrt"
  input: "add_841"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_842/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_842"
  op: "AddV2"
  input: "Sqrt_245"
  input: "add_842/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_248"
  op: "RealDiv"
  input: "add_840"
  input: "add_842"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1344"
  op: "Mul"
  input: "mul_21"
  input: "truediv_248"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1073"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_249"
  op: "Sub"
  input: "ReadVariableOp_1073"
  input: "mul_1344"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_735"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_249"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1074"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_735"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_736"
  op: "AssignVariableOp"
  input: "Mul_1340/ReadVariableOp/Enter"
  input: "add_840"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1075"
  op: "ReadVariableOp"
  input: "Mul_1340/ReadVariableOp/Enter"
  input: "^AssignVariableOp_736"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_737"
  op: "AssignVariableOp"
  input: "Mul_1342/ReadVariableOp/Enter"
  input: "add_841"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1076"
  op: "ReadVariableOp"
  input: "Mul_1342/ReadVariableOp/Enter"
  input: "^AssignVariableOp_737"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1345/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1345/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1345/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1345/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1345"
  op: "Mul"
  input: "Mul_1345/x"
  input: "Mul_1345/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1346/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1346"
  op: "Mul"
  input: "Mul_1346/x"
  input: "CrossReplicaSum_246"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_843"
  op: "AddV2"
  input: "Mul_1345"
  input: "Mul_1346"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1347/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1347/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1347/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1347/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1347"
  op: "Mul"
  input: "Mul_1347/x"
  input: "Mul_1347/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_246"
  op: "Square"
  input: "CrossReplicaSum_246"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1348/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1348"
  op: "Mul"
  input: "Mul_1348/x"
  input: "Square_246"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_844"
  op: "AddV2"
  input: "Mul_1347"
  input: "Mul_1348"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_246"
  op: "Sqrt"
  input: "add_844"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_845/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_845"
  op: "AddV2"
  input: "Sqrt_246"
  input: "add_845/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_249"
  op: "RealDiv"
  input: "add_843"
  input: "add_845"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1349"
  op: "Mul"
  input: "mul_21"
  input: "truediv_249"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1077"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_250"
  op: "Sub"
  input: "ReadVariableOp_1077"
  input: "mul_1349"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_738"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_250"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1078"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_738"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_739"
  op: "AssignVariableOp"
  input: "Mul_1345/ReadVariableOp/Enter"
  input: "add_843"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1079"
  op: "ReadVariableOp"
  input: "Mul_1345/ReadVariableOp/Enter"
  input: "^AssignVariableOp_739"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_740"
  op: "AssignVariableOp"
  input: "Mul_1347/ReadVariableOp/Enter"
  input: "add_844"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1080"
  op: "ReadVariableOp"
  input: "Mul_1347/ReadVariableOp/Enter"
  input: "^AssignVariableOp_740"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1350/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1350/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1350/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1350/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1350"
  op: "Mul"
  input: "Mul_1350/x"
  input: "Mul_1350/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1351/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1351"
  op: "Mul"
  input: "Mul_1351/x"
  input: "CrossReplicaSum_247"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_846"
  op: "AddV2"
  input: "Mul_1350"
  input: "Mul_1351"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1352/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1352/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1352/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1352/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1352"
  op: "Mul"
  input: "Mul_1352/x"
  input: "Mul_1352/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_247"
  op: "Square"
  input: "CrossReplicaSum_247"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1353/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1353"
  op: "Mul"
  input: "Mul_1353/x"
  input: "Square_247"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_847"
  op: "AddV2"
  input: "Mul_1352"
  input: "Mul_1353"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_247"
  op: "Sqrt"
  input: "add_847"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_848/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_848"
  op: "AddV2"
  input: "Sqrt_247"
  input: "add_848/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_250"
  op: "RealDiv"
  input: "add_846"
  input: "add_848"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1354"
  op: "Mul"
  input: "mul_21"
  input: "truediv_250"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1081"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_251"
  op: "Sub"
  input: "ReadVariableOp_1081"
  input: "mul_1354"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_741"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_251"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1082"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_741"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_742"
  op: "AssignVariableOp"
  input: "Mul_1350/ReadVariableOp/Enter"
  input: "add_846"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1083"
  op: "ReadVariableOp"
  input: "Mul_1350/ReadVariableOp/Enter"
  input: "^AssignVariableOp_742"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_743"
  op: "AssignVariableOp"
  input: "Mul_1352/ReadVariableOp/Enter"
  input: "add_847"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1084"
  op: "ReadVariableOp"
  input: "Mul_1352/ReadVariableOp/Enter"
  input: "^AssignVariableOp_743"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1355/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1355/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1355/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1355/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1355"
  op: "Mul"
  input: "Mul_1355/x"
  input: "Mul_1355/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1356/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1356"
  op: "Mul"
  input: "Mul_1356/x"
  input: "CrossReplicaSum_248"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_849"
  op: "AddV2"
  input: "Mul_1355"
  input: "Mul_1356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1357/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1357/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1357/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1357/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1357"
  op: "Mul"
  input: "Mul_1357/x"
  input: "Mul_1357/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_248"
  op: "Square"
  input: "CrossReplicaSum_248"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1358/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1358"
  op: "Mul"
  input: "Mul_1358/x"
  input: "Square_248"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_850"
  op: "AddV2"
  input: "Mul_1357"
  input: "Mul_1358"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_248"
  op: "Sqrt"
  input: "add_850"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_851/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_851"
  op: "AddV2"
  input: "Sqrt_248"
  input: "add_851/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_251"
  op: "RealDiv"
  input: "add_849"
  input: "add_851"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1085"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1359/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1359"
  op: "Mul"
  input: "mul_1359/x"
  input: "ReadVariableOp_1085"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_852"
  op: "AddV2"
  input: "truediv_251"
  input: "mul_1359"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1360"
  op: "Mul"
  input: "mul_21"
  input: "add_852"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1086"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_252"
  op: "Sub"
  input: "ReadVariableOp_1086"
  input: "mul_1360"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_744"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_252"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1087"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_744"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_745"
  op: "AssignVariableOp"
  input: "Mul_1355/ReadVariableOp/Enter"
  input: "add_849"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1088"
  op: "ReadVariableOp"
  input: "Mul_1355/ReadVariableOp/Enter"
  input: "^AssignVariableOp_745"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_746"
  op: "AssignVariableOp"
  input: "Mul_1357/ReadVariableOp/Enter"
  input: "add_850"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1089"
  op: "ReadVariableOp"
  input: "Mul_1357/ReadVariableOp/Enter"
  input: "^AssignVariableOp_746"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1361/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1361/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1361/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1361/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1361"
  op: "Mul"
  input: "Mul_1361/x"
  input: "Mul_1361/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1362/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1362"
  op: "Mul"
  input: "Mul_1362/x"
  input: "CrossReplicaSum_249"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_853"
  op: "AddV2"
  input: "Mul_1361"
  input: "Mul_1362"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1363/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1363/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1363/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1363/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1363"
  op: "Mul"
  input: "Mul_1363/x"
  input: "Mul_1363/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_249"
  op: "Square"
  input: "CrossReplicaSum_249"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1364/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1364"
  op: "Mul"
  input: "Mul_1364/x"
  input: "Square_249"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_854"
  op: "AddV2"
  input: "Mul_1363"
  input: "Mul_1364"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_249"
  op: "Sqrt"
  input: "add_854"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_855/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_855"
  op: "AddV2"
  input: "Sqrt_249"
  input: "add_855/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_252"
  op: "RealDiv"
  input: "add_853"
  input: "add_855"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1365"
  op: "Mul"
  input: "mul_21"
  input: "truediv_252"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1090"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_253"
  op: "Sub"
  input: "ReadVariableOp_1090"
  input: "mul_1365"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_747"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_253"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1091"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_747"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_748"
  op: "AssignVariableOp"
  input: "Mul_1361/ReadVariableOp/Enter"
  input: "add_853"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1092"
  op: "ReadVariableOp"
  input: "Mul_1361/ReadVariableOp/Enter"
  input: "^AssignVariableOp_748"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_749"
  op: "AssignVariableOp"
  input: "Mul_1363/ReadVariableOp/Enter"
  input: "add_854"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1093"
  op: "ReadVariableOp"
  input: "Mul_1363/ReadVariableOp/Enter"
  input: "^AssignVariableOp_749"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1366/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1366/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1366/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1366/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1366"
  op: "Mul"
  input: "Mul_1366/x"
  input: "Mul_1366/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1367/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1367"
  op: "Mul"
  input: "Mul_1367/x"
  input: "CrossReplicaSum_250"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_856"
  op: "AddV2"
  input: "Mul_1366"
  input: "Mul_1367"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1368/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1368/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1368/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1368/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1368"
  op: "Mul"
  input: "Mul_1368/x"
  input: "Mul_1368/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_250"
  op: "Square"
  input: "CrossReplicaSum_250"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1369/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1369"
  op: "Mul"
  input: "Mul_1369/x"
  input: "Square_250"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_857"
  op: "AddV2"
  input: "Mul_1368"
  input: "Mul_1369"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_250"
  op: "Sqrt"
  input: "add_857"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_858/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_858"
  op: "AddV2"
  input: "Sqrt_250"
  input: "add_858/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_253"
  op: "RealDiv"
  input: "add_856"
  input: "add_858"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1094"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1370/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1370"
  op: "Mul"
  input: "mul_1370/x"
  input: "ReadVariableOp_1094"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_859"
  op: "AddV2"
  input: "truediv_253"
  input: "mul_1370"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1371"
  op: "Mul"
  input: "mul_21"
  input: "add_859"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1095"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_254"
  op: "Sub"
  input: "ReadVariableOp_1095"
  input: "mul_1371"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_750"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_254"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1096"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_750"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_751"
  op: "AssignVariableOp"
  input: "Mul_1366/ReadVariableOp/Enter"
  input: "add_856"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1097"
  op: "ReadVariableOp"
  input: "Mul_1366/ReadVariableOp/Enter"
  input: "^AssignVariableOp_751"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_752"
  op: "AssignVariableOp"
  input: "Mul_1368/ReadVariableOp/Enter"
  input: "add_857"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1098"
  op: "ReadVariableOp"
  input: "Mul_1368/ReadVariableOp/Enter"
  input: "^AssignVariableOp_752"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_m"
  input: "electra/encoder/layer_2/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_v"
  input: "electra/encoder/layer_2/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1372/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1372/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1372/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1372/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1372"
  op: "Mul"
  input: "Mul_1372/x"
  input: "Mul_1372/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1373/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1373"
  op: "Mul"
  input: "Mul_1373/x"
  input: "CrossReplicaSum_251"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_860"
  op: "AddV2"
  input: "Mul_1372"
  input: "Mul_1373"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1374/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1374/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1374/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1374/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1374"
  op: "Mul"
  input: "Mul_1374/x"
  input: "Mul_1374/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_251"
  op: "Square"
  input: "CrossReplicaSum_251"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1375/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1375"
  op: "Mul"
  input: "Mul_1375/x"
  input: "Square_251"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_861"
  op: "AddV2"
  input: "Mul_1374"
  input: "Mul_1375"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_251"
  op: "Sqrt"
  input: "add_861"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_862/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_862"
  op: "AddV2"
  input: "Sqrt_251"
  input: "add_862/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_254"
  op: "RealDiv"
  input: "add_860"
  input: "add_862"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1376"
  op: "Mul"
  input: "mul_21"
  input: "truediv_254"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1099"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_255"
  op: "Sub"
  input: "ReadVariableOp_1099"
  input: "mul_1376"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_753"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_255"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1100"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_753"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_754"
  op: "AssignVariableOp"
  input: "Mul_1372/ReadVariableOp/Enter"
  input: "add_860"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1101"
  op: "ReadVariableOp"
  input: "Mul_1372/ReadVariableOp/Enter"
  input: "^AssignVariableOp_754"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_755"
  op: "AssignVariableOp"
  input: "Mul_1374/ReadVariableOp/Enter"
  input: "add_861"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1102"
  op: "ReadVariableOp"
  input: "Mul_1374/ReadVariableOp/Enter"
  input: "^AssignVariableOp_755"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1377/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1377/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1377/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1377/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1377"
  op: "Mul"
  input: "Mul_1377/x"
  input: "Mul_1377/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1378/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1378"
  op: "Mul"
  input: "Mul_1378/x"
  input: "CrossReplicaSum_252"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_863"
  op: "AddV2"
  input: "Mul_1377"
  input: "Mul_1378"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1379/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1379/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1379/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1379/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1379"
  op: "Mul"
  input: "Mul_1379/x"
  input: "Mul_1379/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_252"
  op: "Square"
  input: "CrossReplicaSum_252"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1380/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1380"
  op: "Mul"
  input: "Mul_1380/x"
  input: "Square_252"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_864"
  op: "AddV2"
  input: "Mul_1379"
  input: "Mul_1380"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_252"
  op: "Sqrt"
  input: "add_864"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_865/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_865"
  op: "AddV2"
  input: "Sqrt_252"
  input: "add_865/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_255"
  op: "RealDiv"
  input: "add_863"
  input: "add_865"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1381"
  op: "Mul"
  input: "mul_21"
  input: "truediv_255"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1103"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_256"
  op: "Sub"
  input: "ReadVariableOp_1103"
  input: "mul_1381"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_756"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_256"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1104"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_756"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_757"
  op: "AssignVariableOp"
  input: "Mul_1377/ReadVariableOp/Enter"
  input: "add_863"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1105"
  op: "ReadVariableOp"
  input: "Mul_1377/ReadVariableOp/Enter"
  input: "^AssignVariableOp_757"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_758"
  op: "AssignVariableOp"
  input: "Mul_1379/ReadVariableOp/Enter"
  input: "add_864"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1106"
  op: "ReadVariableOp"
  input: "Mul_1379/ReadVariableOp/Enter"
  input: "^AssignVariableOp_758"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1382/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1382/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1382/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1382/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1382"
  op: "Mul"
  input: "Mul_1382/x"
  input: "Mul_1382/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1383/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1383"
  op: "Mul"
  input: "Mul_1383/x"
  input: "CrossReplicaSum_253"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_866"
  op: "AddV2"
  input: "Mul_1382"
  input: "Mul_1383"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1384/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1384/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1384/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1384/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1384"
  op: "Mul"
  input: "Mul_1384/x"
  input: "Mul_1384/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_253"
  op: "Square"
  input: "CrossReplicaSum_253"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1385/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1385"
  op: "Mul"
  input: "Mul_1385/x"
  input: "Square_253"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_867"
  op: "AddV2"
  input: "Mul_1384"
  input: "Mul_1385"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_253"
  op: "Sqrt"
  input: "add_867"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_868/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_868"
  op: "AddV2"
  input: "Sqrt_253"
  input: "add_868/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_256"
  op: "RealDiv"
  input: "add_866"
  input: "add_868"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1386"
  op: "Mul"
  input: "mul_21"
  input: "truediv_256"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1107"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_257"
  op: "Sub"
  input: "ReadVariableOp_1107"
  input: "mul_1386"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_759"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_257"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1108"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_759"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_760"
  op: "AssignVariableOp"
  input: "Mul_1382/ReadVariableOp/Enter"
  input: "add_866"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1109"
  op: "ReadVariableOp"
  input: "Mul_1382/ReadVariableOp/Enter"
  input: "^AssignVariableOp_760"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_761"
  op: "AssignVariableOp"
  input: "Mul_1384/ReadVariableOp/Enter"
  input: "add_867"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1110"
  op: "ReadVariableOp"
  input: "Mul_1384/ReadVariableOp/Enter"
  input: "^AssignVariableOp_761"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1387/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1387/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1387/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1387/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1387"
  op: "Mul"
  input: "Mul_1387/x"
  input: "Mul_1387/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1388/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1388"
  op: "Mul"
  input: "Mul_1388/x"
  input: "CrossReplicaSum_254"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_869"
  op: "AddV2"
  input: "Mul_1387"
  input: "Mul_1388"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1389/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1389/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1389/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1389/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1389"
  op: "Mul"
  input: "Mul_1389/x"
  input: "Mul_1389/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_254"
  op: "Square"
  input: "CrossReplicaSum_254"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1390/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1390"
  op: "Mul"
  input: "Mul_1390/x"
  input: "Square_254"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_870"
  op: "AddV2"
  input: "Mul_1389"
  input: "Mul_1390"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_254"
  op: "Sqrt"
  input: "add_870"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_871/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_871"
  op: "AddV2"
  input: "Sqrt_254"
  input: "add_871/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_257"
  op: "RealDiv"
  input: "add_869"
  input: "add_871"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1111"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1391/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1391"
  op: "Mul"
  input: "mul_1391/x"
  input: "ReadVariableOp_1111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_872"
  op: "AddV2"
  input: "truediv_257"
  input: "mul_1391"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1392"
  op: "Mul"
  input: "mul_21"
  input: "add_872"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1112"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_258"
  op: "Sub"
  input: "ReadVariableOp_1112"
  input: "mul_1392"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_762"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_258"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1113"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_762"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_763"
  op: "AssignVariableOp"
  input: "Mul_1387/ReadVariableOp/Enter"
  input: "add_869"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1114"
  op: "ReadVariableOp"
  input: "Mul_1387/ReadVariableOp/Enter"
  input: "^AssignVariableOp_763"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_764"
  op: "AssignVariableOp"
  input: "Mul_1389/ReadVariableOp/Enter"
  input: "add_870"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1115"
  op: "ReadVariableOp"
  input: "Mul_1389/ReadVariableOp/Enter"
  input: "^AssignVariableOp_764"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1393/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1393/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1393/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1393/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1393"
  op: "Mul"
  input: "Mul_1393/x"
  input: "Mul_1393/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1394/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1394"
  op: "Mul"
  input: "Mul_1394/x"
  input: "CrossReplicaSum_255"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_873"
  op: "AddV2"
  input: "Mul_1393"
  input: "Mul_1394"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1395/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1395/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1395/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1395/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1395"
  op: "Mul"
  input: "Mul_1395/x"
  input: "Mul_1395/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_255"
  op: "Square"
  input: "CrossReplicaSum_255"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1396/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1396"
  op: "Mul"
  input: "Mul_1396/x"
  input: "Square_255"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_874"
  op: "AddV2"
  input: "Mul_1395"
  input: "Mul_1396"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_255"
  op: "Sqrt"
  input: "add_874"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_875/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_875"
  op: "AddV2"
  input: "Sqrt_255"
  input: "add_875/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_258"
  op: "RealDiv"
  input: "add_873"
  input: "add_875"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1397"
  op: "Mul"
  input: "mul_21"
  input: "truediv_258"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1116"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_259"
  op: "Sub"
  input: "ReadVariableOp_1116"
  input: "mul_1397"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_765"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_259"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1117"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_765"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_766"
  op: "AssignVariableOp"
  input: "Mul_1393/ReadVariableOp/Enter"
  input: "add_873"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1118"
  op: "ReadVariableOp"
  input: "Mul_1393/ReadVariableOp/Enter"
  input: "^AssignVariableOp_766"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_767"
  op: "AssignVariableOp"
  input: "Mul_1395/ReadVariableOp/Enter"
  input: "add_874"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1119"
  op: "ReadVariableOp"
  input: "Mul_1395/ReadVariableOp/Enter"
  input: "^AssignVariableOp_767"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1398/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1398/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1398/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1398/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1398"
  op: "Mul"
  input: "Mul_1398/x"
  input: "Mul_1398/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1399/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1399"
  op: "Mul"
  input: "Mul_1399/x"
  input: "CrossReplicaSum_256"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_876"
  op: "AddV2"
  input: "Mul_1398"
  input: "Mul_1399"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1400/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1400/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1400/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1400/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1400"
  op: "Mul"
  input: "Mul_1400/x"
  input: "Mul_1400/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_256"
  op: "Square"
  input: "CrossReplicaSum_256"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1401/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1401"
  op: "Mul"
  input: "Mul_1401/x"
  input: "Square_256"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_877"
  op: "AddV2"
  input: "Mul_1400"
  input: "Mul_1401"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_256"
  op: "Sqrt"
  input: "add_877"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_878/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_878"
  op: "AddV2"
  input: "Sqrt_256"
  input: "add_878/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_259"
  op: "RealDiv"
  input: "add_876"
  input: "add_878"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1120"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1402/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1402"
  op: "Mul"
  input: "mul_1402/x"
  input: "ReadVariableOp_1120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_879"
  op: "AddV2"
  input: "truediv_259"
  input: "mul_1402"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1403"
  op: "Mul"
  input: "mul_21"
  input: "add_879"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1121"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_260"
  op: "Sub"
  input: "ReadVariableOp_1121"
  input: "mul_1403"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_768"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_260"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1122"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_768"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_769"
  op: "AssignVariableOp"
  input: "Mul_1398/ReadVariableOp/Enter"
  input: "add_876"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1123"
  op: "ReadVariableOp"
  input: "Mul_1398/ReadVariableOp/Enter"
  input: "^AssignVariableOp_769"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_770"
  op: "AssignVariableOp"
  input: "Mul_1400/ReadVariableOp/Enter"
  input: "add_877"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1124"
  op: "ReadVariableOp"
  input: "Mul_1400/ReadVariableOp/Enter"
  input: "^AssignVariableOp_770"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1404/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1404/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1404/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1404/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1404"
  op: "Mul"
  input: "Mul_1404/x"
  input: "Mul_1404/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1405/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1405"
  op: "Mul"
  input: "Mul_1405/x"
  input: "CrossReplicaSum_257"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_880"
  op: "AddV2"
  input: "Mul_1404"
  input: "Mul_1405"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1406/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1406/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1406/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1406/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1406"
  op: "Mul"
  input: "Mul_1406/x"
  input: "Mul_1406/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_257"
  op: "Square"
  input: "CrossReplicaSum_257"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1407/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1407"
  op: "Mul"
  input: "Mul_1407/x"
  input: "Square_257"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_881"
  op: "AddV2"
  input: "Mul_1406"
  input: "Mul_1407"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_257"
  op: "Sqrt"
  input: "add_881"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_882/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_882"
  op: "AddV2"
  input: "Sqrt_257"
  input: "add_882/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_260"
  op: "RealDiv"
  input: "add_880"
  input: "add_882"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1408"
  op: "Mul"
  input: "mul_21"
  input: "truediv_260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1125"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_261"
  op: "Sub"
  input: "ReadVariableOp_1125"
  input: "mul_1408"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_771"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_261"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1126"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_771"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_772"
  op: "AssignVariableOp"
  input: "Mul_1404/ReadVariableOp/Enter"
  input: "add_880"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1127"
  op: "ReadVariableOp"
  input: "Mul_1404/ReadVariableOp/Enter"
  input: "^AssignVariableOp_772"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_773"
  op: "AssignVariableOp"
  input: "Mul_1406/ReadVariableOp/Enter"
  input: "add_881"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1128"
  op: "ReadVariableOp"
  input: "Mul_1406/ReadVariableOp/Enter"
  input: "^AssignVariableOp_773"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1409/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1409/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1409/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1409/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1409"
  op: "Mul"
  input: "Mul_1409/x"
  input: "Mul_1409/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1410/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1410"
  op: "Mul"
  input: "Mul_1410/x"
  input: "CrossReplicaSum_258"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_883"
  op: "AddV2"
  input: "Mul_1409"
  input: "Mul_1410"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1411/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1411/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1411/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1411/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1411"
  op: "Mul"
  input: "Mul_1411/x"
  input: "Mul_1411/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_258"
  op: "Square"
  input: "CrossReplicaSum_258"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1412/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1412"
  op: "Mul"
  input: "Mul_1412/x"
  input: "Square_258"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_884"
  op: "AddV2"
  input: "Mul_1411"
  input: "Mul_1412"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_258"
  op: "Sqrt"
  input: "add_884"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_885/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_885"
  op: "AddV2"
  input: "Sqrt_258"
  input: "add_885/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_261"
  op: "RealDiv"
  input: "add_883"
  input: "add_885"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1129"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1413/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1413"
  op: "Mul"
  input: "mul_1413/x"
  input: "ReadVariableOp_1129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_886"
  op: "AddV2"
  input: "truediv_261"
  input: "mul_1413"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1414"
  op: "Mul"
  input: "mul_21"
  input: "add_886"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1130"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_262"
  op: "Sub"
  input: "ReadVariableOp_1130"
  input: "mul_1414"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_774"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_262"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1131"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_774"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_775"
  op: "AssignVariableOp"
  input: "Mul_1409/ReadVariableOp/Enter"
  input: "add_883"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1132"
  op: "ReadVariableOp"
  input: "Mul_1409/ReadVariableOp/Enter"
  input: "^AssignVariableOp_775"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_776"
  op: "AssignVariableOp"
  input: "Mul_1411/ReadVariableOp/Enter"
  input: "add_884"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1133"
  op: "ReadVariableOp"
  input: "Mul_1411/ReadVariableOp/Enter"
  input: "^AssignVariableOp_776"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1415/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1415/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1415/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1415/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1415"
  op: "Mul"
  input: "Mul_1415/x"
  input: "Mul_1415/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1416/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1416"
  op: "Mul"
  input: "Mul_1416/x"
  input: "CrossReplicaSum_259"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_887"
  op: "AddV2"
  input: "Mul_1415"
  input: "Mul_1416"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1417/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1417/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1417/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1417/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1417"
  op: "Mul"
  input: "Mul_1417/x"
  input: "Mul_1417/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_259"
  op: "Square"
  input: "CrossReplicaSum_259"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1418/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1418"
  op: "Mul"
  input: "Mul_1418/x"
  input: "Square_259"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_888"
  op: "AddV2"
  input: "Mul_1417"
  input: "Mul_1418"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_259"
  op: "Sqrt"
  input: "add_888"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_889/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_889"
  op: "AddV2"
  input: "Sqrt_259"
  input: "add_889/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_262"
  op: "RealDiv"
  input: "add_887"
  input: "add_889"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1419"
  op: "Mul"
  input: "mul_21"
  input: "truediv_262"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1134"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_263"
  op: "Sub"
  input: "ReadVariableOp_1134"
  input: "mul_1419"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_777"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_263"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1135"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_777"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_778"
  op: "AssignVariableOp"
  input: "Mul_1415/ReadVariableOp/Enter"
  input: "add_887"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1136"
  op: "ReadVariableOp"
  input: "Mul_1415/ReadVariableOp/Enter"
  input: "^AssignVariableOp_778"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_779"
  op: "AssignVariableOp"
  input: "Mul_1417/ReadVariableOp/Enter"
  input: "add_888"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1137"
  op: "ReadVariableOp"
  input: "Mul_1417/ReadVariableOp/Enter"
  input: "^AssignVariableOp_779"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1420/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1420/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1420/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1420/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1420"
  op: "Mul"
  input: "Mul_1420/x"
  input: "Mul_1420/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1421/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1421"
  op: "Mul"
  input: "Mul_1421/x"
  input: "CrossReplicaSum_260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_890"
  op: "AddV2"
  input: "Mul_1420"
  input: "Mul_1421"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1422/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1422/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1422/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1422/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1422"
  op: "Mul"
  input: "Mul_1422/x"
  input: "Mul_1422/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_260"
  op: "Square"
  input: "CrossReplicaSum_260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1423/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1423"
  op: "Mul"
  input: "Mul_1423/x"
  input: "Square_260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_891"
  op: "AddV2"
  input: "Mul_1422"
  input: "Mul_1423"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_260"
  op: "Sqrt"
  input: "add_891"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_892/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_892"
  op: "AddV2"
  input: "Sqrt_260"
  input: "add_892/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_263"
  op: "RealDiv"
  input: "add_890"
  input: "add_892"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1138"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1424/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1424"
  op: "Mul"
  input: "mul_1424/x"
  input: "ReadVariableOp_1138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_893"
  op: "AddV2"
  input: "truediv_263"
  input: "mul_1424"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1425"
  op: "Mul"
  input: "mul_21"
  input: "add_893"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1139"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_264"
  op: "Sub"
  input: "ReadVariableOp_1139"
  input: "mul_1425"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_780"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_264"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1140"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_780"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_781"
  op: "AssignVariableOp"
  input: "Mul_1420/ReadVariableOp/Enter"
  input: "add_890"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1141"
  op: "ReadVariableOp"
  input: "Mul_1420/ReadVariableOp/Enter"
  input: "^AssignVariableOp_781"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_782"
  op: "AssignVariableOp"
  input: "Mul_1422/ReadVariableOp/Enter"
  input: "add_891"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1142"
  op: "ReadVariableOp"
  input: "Mul_1422/ReadVariableOp/Enter"
  input: "^AssignVariableOp_782"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1426/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1426/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1426/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1426/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1426"
  op: "Mul"
  input: "Mul_1426/x"
  input: "Mul_1426/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1427/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1427"
  op: "Mul"
  input: "Mul_1427/x"
  input: "CrossReplicaSum_261"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_894"
  op: "AddV2"
  input: "Mul_1426"
  input: "Mul_1427"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1428/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1428/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1428/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1428/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1428"
  op: "Mul"
  input: "Mul_1428/x"
  input: "Mul_1428/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_261"
  op: "Square"
  input: "CrossReplicaSum_261"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1429/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1429"
  op: "Mul"
  input: "Mul_1429/x"
  input: "Square_261"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_895"
  op: "AddV2"
  input: "Mul_1428"
  input: "Mul_1429"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_261"
  op: "Sqrt"
  input: "add_895"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_896/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_896"
  op: "AddV2"
  input: "Sqrt_261"
  input: "add_896/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_264"
  op: "RealDiv"
  input: "add_894"
  input: "add_896"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1430"
  op: "Mul"
  input: "mul_21"
  input: "truediv_264"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1143"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_265"
  op: "Sub"
  input: "ReadVariableOp_1143"
  input: "mul_1430"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_783"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_265"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1144"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_783"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_784"
  op: "AssignVariableOp"
  input: "Mul_1426/ReadVariableOp/Enter"
  input: "add_894"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1145"
  op: "ReadVariableOp"
  input: "Mul_1426/ReadVariableOp/Enter"
  input: "^AssignVariableOp_784"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_785"
  op: "AssignVariableOp"
  input: "Mul_1428/ReadVariableOp/Enter"
  input: "add_895"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1146"
  op: "ReadVariableOp"
  input: "Mul_1428/ReadVariableOp/Enter"
  input: "^AssignVariableOp_785"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1431/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1431/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1431/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1431/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1431"
  op: "Mul"
  input: "Mul_1431/x"
  input: "Mul_1431/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1432/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1432"
  op: "Mul"
  input: "Mul_1432/x"
  input: "CrossReplicaSum_262"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_897"
  op: "AddV2"
  input: "Mul_1431"
  input: "Mul_1432"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1433/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1433/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1433/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1433/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1433"
  op: "Mul"
  input: "Mul_1433/x"
  input: "Mul_1433/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_262"
  op: "Square"
  input: "CrossReplicaSum_262"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1434/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1434"
  op: "Mul"
  input: "Mul_1434/x"
  input: "Square_262"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_898"
  op: "AddV2"
  input: "Mul_1433"
  input: "Mul_1434"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_262"
  op: "Sqrt"
  input: "add_898"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_899/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_899"
  op: "AddV2"
  input: "Sqrt_262"
  input: "add_899/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_265"
  op: "RealDiv"
  input: "add_897"
  input: "add_899"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1435"
  op: "Mul"
  input: "mul_21"
  input: "truediv_265"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1147"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_266"
  op: "Sub"
  input: "ReadVariableOp_1147"
  input: "mul_1435"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_786"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_266"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1148"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_786"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_787"
  op: "AssignVariableOp"
  input: "Mul_1431/ReadVariableOp/Enter"
  input: "add_897"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1149"
  op: "ReadVariableOp"
  input: "Mul_1431/ReadVariableOp/Enter"
  input: "^AssignVariableOp_787"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_788"
  op: "AssignVariableOp"
  input: "Mul_1433/ReadVariableOp/Enter"
  input: "add_898"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1150"
  op: "ReadVariableOp"
  input: "Mul_1433/ReadVariableOp/Enter"
  input: "^AssignVariableOp_788"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1436/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1436/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1436/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1436/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1436"
  op: "Mul"
  input: "Mul_1436/x"
  input: "Mul_1436/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1437/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1437"
  op: "Mul"
  input: "Mul_1437/x"
  input: "CrossReplicaSum_263"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_900"
  op: "AddV2"
  input: "Mul_1436"
  input: "Mul_1437"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1438/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1438/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1438/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1438/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1438"
  op: "Mul"
  input: "Mul_1438/x"
  input: "Mul_1438/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_263"
  op: "Square"
  input: "CrossReplicaSum_263"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1439/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1439"
  op: "Mul"
  input: "Mul_1439/x"
  input: "Square_263"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_901"
  op: "AddV2"
  input: "Mul_1438"
  input: "Mul_1439"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_263"
  op: "Sqrt"
  input: "add_901"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_902/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_902"
  op: "AddV2"
  input: "Sqrt_263"
  input: "add_902/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_266"
  op: "RealDiv"
  input: "add_900"
  input: "add_902"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1440"
  op: "Mul"
  input: "mul_21"
  input: "truediv_266"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1151"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_267"
  op: "Sub"
  input: "ReadVariableOp_1151"
  input: "mul_1440"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_789"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_267"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1152"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_789"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_790"
  op: "AssignVariableOp"
  input: "Mul_1436/ReadVariableOp/Enter"
  input: "add_900"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1153"
  op: "ReadVariableOp"
  input: "Mul_1436/ReadVariableOp/Enter"
  input: "^AssignVariableOp_790"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_791"
  op: "AssignVariableOp"
  input: "Mul_1438/ReadVariableOp/Enter"
  input: "add_901"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1154"
  op: "ReadVariableOp"
  input: "Mul_1438/ReadVariableOp/Enter"
  input: "^AssignVariableOp_791"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1441/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1441/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1441/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1441/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1441"
  op: "Mul"
  input: "Mul_1441/x"
  input: "Mul_1441/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1442/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1442"
  op: "Mul"
  input: "Mul_1442/x"
  input: "CrossReplicaSum_264"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_903"
  op: "AddV2"
  input: "Mul_1441"
  input: "Mul_1442"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1443/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1443/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1443/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1443/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1443"
  op: "Mul"
  input: "Mul_1443/x"
  input: "Mul_1443/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_264"
  op: "Square"
  input: "CrossReplicaSum_264"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1444/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1444"
  op: "Mul"
  input: "Mul_1444/x"
  input: "Square_264"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_904"
  op: "AddV2"
  input: "Mul_1443"
  input: "Mul_1444"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_264"
  op: "Sqrt"
  input: "add_904"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_905/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_905"
  op: "AddV2"
  input: "Sqrt_264"
  input: "add_905/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_267"
  op: "RealDiv"
  input: "add_903"
  input: "add_905"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1155"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1445/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1445"
  op: "Mul"
  input: "mul_1445/x"
  input: "ReadVariableOp_1155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_906"
  op: "AddV2"
  input: "truediv_267"
  input: "mul_1445"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1446"
  op: "Mul"
  input: "mul_21"
  input: "add_906"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1156"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_268"
  op: "Sub"
  input: "ReadVariableOp_1156"
  input: "mul_1446"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_792"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_268"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1157"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_792"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_793"
  op: "AssignVariableOp"
  input: "Mul_1441/ReadVariableOp/Enter"
  input: "add_903"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1158"
  op: "ReadVariableOp"
  input: "Mul_1441/ReadVariableOp/Enter"
  input: "^AssignVariableOp_793"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_794"
  op: "AssignVariableOp"
  input: "Mul_1443/ReadVariableOp/Enter"
  input: "add_904"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1159"
  op: "ReadVariableOp"
  input: "Mul_1443/ReadVariableOp/Enter"
  input: "^AssignVariableOp_794"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1447/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1447/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1447/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1447/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1447"
  op: "Mul"
  input: "Mul_1447/x"
  input: "Mul_1447/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1448/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1448"
  op: "Mul"
  input: "Mul_1448/x"
  input: "CrossReplicaSum_265"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_907"
  op: "AddV2"
  input: "Mul_1447"
  input: "Mul_1448"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1449/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1449/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1449/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1449/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1449"
  op: "Mul"
  input: "Mul_1449/x"
  input: "Mul_1449/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_265"
  op: "Square"
  input: "CrossReplicaSum_265"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1450/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1450"
  op: "Mul"
  input: "Mul_1450/x"
  input: "Square_265"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_908"
  op: "AddV2"
  input: "Mul_1449"
  input: "Mul_1450"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_265"
  op: "Sqrt"
  input: "add_908"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_909/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_909"
  op: "AddV2"
  input: "Sqrt_265"
  input: "add_909/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_268"
  op: "RealDiv"
  input: "add_907"
  input: "add_909"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1451"
  op: "Mul"
  input: "mul_21"
  input: "truediv_268"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1160"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_269"
  op: "Sub"
  input: "ReadVariableOp_1160"
  input: "mul_1451"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_795"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_269"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1161"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_795"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_796"
  op: "AssignVariableOp"
  input: "Mul_1447/ReadVariableOp/Enter"
  input: "add_907"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1162"
  op: "ReadVariableOp"
  input: "Mul_1447/ReadVariableOp/Enter"
  input: "^AssignVariableOp_796"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_797"
  op: "AssignVariableOp"
  input: "Mul_1449/ReadVariableOp/Enter"
  input: "add_908"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1163"
  op: "ReadVariableOp"
  input: "Mul_1449/ReadVariableOp/Enter"
  input: "^AssignVariableOp_797"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1452/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1452/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1452/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1452/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1452"
  op: "Mul"
  input: "Mul_1452/x"
  input: "Mul_1452/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1453/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1453"
  op: "Mul"
  input: "Mul_1453/x"
  input: "CrossReplicaSum_266"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_910"
  op: "AddV2"
  input: "Mul_1452"
  input: "Mul_1453"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1454/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1454/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1454/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1454/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1454"
  op: "Mul"
  input: "Mul_1454/x"
  input: "Mul_1454/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_266"
  op: "Square"
  input: "CrossReplicaSum_266"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1455/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1455"
  op: "Mul"
  input: "Mul_1455/x"
  input: "Square_266"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_911"
  op: "AddV2"
  input: "Mul_1454"
  input: "Mul_1455"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_266"
  op: "Sqrt"
  input: "add_911"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_912/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_912"
  op: "AddV2"
  input: "Sqrt_266"
  input: "add_912/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_269"
  op: "RealDiv"
  input: "add_910"
  input: "add_912"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1164"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1456/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1456"
  op: "Mul"
  input: "mul_1456/x"
  input: "ReadVariableOp_1164"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_913"
  op: "AddV2"
  input: "truediv_269"
  input: "mul_1456"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1457"
  op: "Mul"
  input: "mul_21"
  input: "add_913"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1165"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_270"
  op: "Sub"
  input: "ReadVariableOp_1165"
  input: "mul_1457"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_798"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_270"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1166"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_798"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_799"
  op: "AssignVariableOp"
  input: "Mul_1452/ReadVariableOp/Enter"
  input: "add_910"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1167"
  op: "ReadVariableOp"
  input: "Mul_1452/ReadVariableOp/Enter"
  input: "^AssignVariableOp_799"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_800"
  op: "AssignVariableOp"
  input: "Mul_1454/ReadVariableOp/Enter"
  input: "add_911"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1168"
  op: "ReadVariableOp"
  input: "Mul_1454/ReadVariableOp/Enter"
  input: "^AssignVariableOp_800"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_m"
  input: "electra/encoder/layer_3/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_v"
  input: "electra/encoder/layer_3/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1458/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1458/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1458/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1458/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1458"
  op: "Mul"
  input: "Mul_1458/x"
  input: "Mul_1458/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1459/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1459"
  op: "Mul"
  input: "Mul_1459/x"
  input: "CrossReplicaSum_267"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_914"
  op: "AddV2"
  input: "Mul_1458"
  input: "Mul_1459"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1460/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1460/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1460/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1460/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1460"
  op: "Mul"
  input: "Mul_1460/x"
  input: "Mul_1460/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_267"
  op: "Square"
  input: "CrossReplicaSum_267"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1461/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1461"
  op: "Mul"
  input: "Mul_1461/x"
  input: "Square_267"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_915"
  op: "AddV2"
  input: "Mul_1460"
  input: "Mul_1461"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_267"
  op: "Sqrt"
  input: "add_915"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_916/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_916"
  op: "AddV2"
  input: "Sqrt_267"
  input: "add_916/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_270"
  op: "RealDiv"
  input: "add_914"
  input: "add_916"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1462"
  op: "Mul"
  input: "mul_21"
  input: "truediv_270"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1169"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_271"
  op: "Sub"
  input: "ReadVariableOp_1169"
  input: "mul_1462"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_801"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_271"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1170"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_801"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_802"
  op: "AssignVariableOp"
  input: "Mul_1458/ReadVariableOp/Enter"
  input: "add_914"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1171"
  op: "ReadVariableOp"
  input: "Mul_1458/ReadVariableOp/Enter"
  input: "^AssignVariableOp_802"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_803"
  op: "AssignVariableOp"
  input: "Mul_1460/ReadVariableOp/Enter"
  input: "add_915"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1172"
  op: "ReadVariableOp"
  input: "Mul_1460/ReadVariableOp/Enter"
  input: "^AssignVariableOp_803"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1463/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1463/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1463/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1463/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1463"
  op: "Mul"
  input: "Mul_1463/x"
  input: "Mul_1463/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1464/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1464"
  op: "Mul"
  input: "Mul_1464/x"
  input: "CrossReplicaSum_268"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_917"
  op: "AddV2"
  input: "Mul_1463"
  input: "Mul_1464"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1465/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1465/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1465/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1465/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1465"
  op: "Mul"
  input: "Mul_1465/x"
  input: "Mul_1465/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_268"
  op: "Square"
  input: "CrossReplicaSum_268"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1466/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1466"
  op: "Mul"
  input: "Mul_1466/x"
  input: "Square_268"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_918"
  op: "AddV2"
  input: "Mul_1465"
  input: "Mul_1466"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_268"
  op: "Sqrt"
  input: "add_918"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_919/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_919"
  op: "AddV2"
  input: "Sqrt_268"
  input: "add_919/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_271"
  op: "RealDiv"
  input: "add_917"
  input: "add_919"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1467"
  op: "Mul"
  input: "mul_21"
  input: "truediv_271"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1173"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_272"
  op: "Sub"
  input: "ReadVariableOp_1173"
  input: "mul_1467"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_804"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_272"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1174"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_804"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_805"
  op: "AssignVariableOp"
  input: "Mul_1463/ReadVariableOp/Enter"
  input: "add_917"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1175"
  op: "ReadVariableOp"
  input: "Mul_1463/ReadVariableOp/Enter"
  input: "^AssignVariableOp_805"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_806"
  op: "AssignVariableOp"
  input: "Mul_1465/ReadVariableOp/Enter"
  input: "add_918"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1176"
  op: "ReadVariableOp"
  input: "Mul_1465/ReadVariableOp/Enter"
  input: "^AssignVariableOp_806"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1468/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1468/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1468/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1468/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1468"
  op: "Mul"
  input: "Mul_1468/x"
  input: "Mul_1468/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1469/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1469"
  op: "Mul"
  input: "Mul_1469/x"
  input: "CrossReplicaSum_269"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_920"
  op: "AddV2"
  input: "Mul_1468"
  input: "Mul_1469"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1470/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1470/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1470/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1470/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1470"
  op: "Mul"
  input: "Mul_1470/x"
  input: "Mul_1470/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_269"
  op: "Square"
  input: "CrossReplicaSum_269"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1471/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1471"
  op: "Mul"
  input: "Mul_1471/x"
  input: "Square_269"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_921"
  op: "AddV2"
  input: "Mul_1470"
  input: "Mul_1471"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_269"
  op: "Sqrt"
  input: "add_921"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_922/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_922"
  op: "AddV2"
  input: "Sqrt_269"
  input: "add_922/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_272"
  op: "RealDiv"
  input: "add_920"
  input: "add_922"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1472"
  op: "Mul"
  input: "mul_21"
  input: "truediv_272"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1177"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_273"
  op: "Sub"
  input: "ReadVariableOp_1177"
  input: "mul_1472"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_807"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_273"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1178"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_807"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_808"
  op: "AssignVariableOp"
  input: "Mul_1468/ReadVariableOp/Enter"
  input: "add_920"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1179"
  op: "ReadVariableOp"
  input: "Mul_1468/ReadVariableOp/Enter"
  input: "^AssignVariableOp_808"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_809"
  op: "AssignVariableOp"
  input: "Mul_1470/ReadVariableOp/Enter"
  input: "add_921"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1180"
  op: "ReadVariableOp"
  input: "Mul_1470/ReadVariableOp/Enter"
  input: "^AssignVariableOp_809"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1473/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1473/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1473/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1473/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1473"
  op: "Mul"
  input: "Mul_1473/x"
  input: "Mul_1473/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1474/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1474"
  op: "Mul"
  input: "Mul_1474/x"
  input: "CrossReplicaSum_270"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_923"
  op: "AddV2"
  input: "Mul_1473"
  input: "Mul_1474"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1475/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1475/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1475/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1475/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1475"
  op: "Mul"
  input: "Mul_1475/x"
  input: "Mul_1475/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_270"
  op: "Square"
  input: "CrossReplicaSum_270"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1476/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1476"
  op: "Mul"
  input: "Mul_1476/x"
  input: "Square_270"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_924"
  op: "AddV2"
  input: "Mul_1475"
  input: "Mul_1476"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_270"
  op: "Sqrt"
  input: "add_924"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_925/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_925"
  op: "AddV2"
  input: "Sqrt_270"
  input: "add_925/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_273"
  op: "RealDiv"
  input: "add_923"
  input: "add_925"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1181"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1477/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1477"
  op: "Mul"
  input: "mul_1477/x"
  input: "ReadVariableOp_1181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_926"
  op: "AddV2"
  input: "truediv_273"
  input: "mul_1477"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1478"
  op: "Mul"
  input: "mul_21"
  input: "add_926"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1182"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_274"
  op: "Sub"
  input: "ReadVariableOp_1182"
  input: "mul_1478"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_810"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_274"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1183"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_810"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_811"
  op: "AssignVariableOp"
  input: "Mul_1473/ReadVariableOp/Enter"
  input: "add_923"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1184"
  op: "ReadVariableOp"
  input: "Mul_1473/ReadVariableOp/Enter"
  input: "^AssignVariableOp_811"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_812"
  op: "AssignVariableOp"
  input: "Mul_1475/ReadVariableOp/Enter"
  input: "add_924"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1185"
  op: "ReadVariableOp"
  input: "Mul_1475/ReadVariableOp/Enter"
  input: "^AssignVariableOp_812"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1479/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1479/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1479/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1479/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1479"
  op: "Mul"
  input: "Mul_1479/x"
  input: "Mul_1479/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1480/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1480"
  op: "Mul"
  input: "Mul_1480/x"
  input: "CrossReplicaSum_271"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_927"
  op: "AddV2"
  input: "Mul_1479"
  input: "Mul_1480"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1481/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1481/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1481/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1481/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1481"
  op: "Mul"
  input: "Mul_1481/x"
  input: "Mul_1481/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_271"
  op: "Square"
  input: "CrossReplicaSum_271"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1482/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1482"
  op: "Mul"
  input: "Mul_1482/x"
  input: "Square_271"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_928"
  op: "AddV2"
  input: "Mul_1481"
  input: "Mul_1482"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_271"
  op: "Sqrt"
  input: "add_928"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_929/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_929"
  op: "AddV2"
  input: "Sqrt_271"
  input: "add_929/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_274"
  op: "RealDiv"
  input: "add_927"
  input: "add_929"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1483"
  op: "Mul"
  input: "mul_21"
  input: "truediv_274"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1186"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_275"
  op: "Sub"
  input: "ReadVariableOp_1186"
  input: "mul_1483"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_813"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_275"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1187"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_813"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_814"
  op: "AssignVariableOp"
  input: "Mul_1479/ReadVariableOp/Enter"
  input: "add_927"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1188"
  op: "ReadVariableOp"
  input: "Mul_1479/ReadVariableOp/Enter"
  input: "^AssignVariableOp_814"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_815"
  op: "AssignVariableOp"
  input: "Mul_1481/ReadVariableOp/Enter"
  input: "add_928"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1189"
  op: "ReadVariableOp"
  input: "Mul_1481/ReadVariableOp/Enter"
  input: "^AssignVariableOp_815"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1484/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1484/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1484/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1484/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1484"
  op: "Mul"
  input: "Mul_1484/x"
  input: "Mul_1484/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1485/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1485"
  op: "Mul"
  input: "Mul_1485/x"
  input: "CrossReplicaSum_272"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_930"
  op: "AddV2"
  input: "Mul_1484"
  input: "Mul_1485"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1486/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1486/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1486/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1486/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1486"
  op: "Mul"
  input: "Mul_1486/x"
  input: "Mul_1486/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_272"
  op: "Square"
  input: "CrossReplicaSum_272"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1487/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1487"
  op: "Mul"
  input: "Mul_1487/x"
  input: "Square_272"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_931"
  op: "AddV2"
  input: "Mul_1486"
  input: "Mul_1487"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_272"
  op: "Sqrt"
  input: "add_931"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_932/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_932"
  op: "AddV2"
  input: "Sqrt_272"
  input: "add_932/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_275"
  op: "RealDiv"
  input: "add_930"
  input: "add_932"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1190"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1488/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1488"
  op: "Mul"
  input: "mul_1488/x"
  input: "ReadVariableOp_1190"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_933"
  op: "AddV2"
  input: "truediv_275"
  input: "mul_1488"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1489"
  op: "Mul"
  input: "mul_21"
  input: "add_933"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1191"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_276"
  op: "Sub"
  input: "ReadVariableOp_1191"
  input: "mul_1489"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_816"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_276"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1192"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_816"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_817"
  op: "AssignVariableOp"
  input: "Mul_1484/ReadVariableOp/Enter"
  input: "add_930"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1193"
  op: "ReadVariableOp"
  input: "Mul_1484/ReadVariableOp/Enter"
  input: "^AssignVariableOp_817"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_818"
  op: "AssignVariableOp"
  input: "Mul_1486/ReadVariableOp/Enter"
  input: "add_931"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1194"
  op: "ReadVariableOp"
  input: "Mul_1486/ReadVariableOp/Enter"
  input: "^AssignVariableOp_818"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1490/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1490/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1490/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1490/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1490"
  op: "Mul"
  input: "Mul_1490/x"
  input: "Mul_1490/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1491/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1491"
  op: "Mul"
  input: "Mul_1491/x"
  input: "CrossReplicaSum_273"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_934"
  op: "AddV2"
  input: "Mul_1490"
  input: "Mul_1491"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1492/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1492/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1492/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1492/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1492"
  op: "Mul"
  input: "Mul_1492/x"
  input: "Mul_1492/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_273"
  op: "Square"
  input: "CrossReplicaSum_273"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1493/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1493"
  op: "Mul"
  input: "Mul_1493/x"
  input: "Square_273"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_935"
  op: "AddV2"
  input: "Mul_1492"
  input: "Mul_1493"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_273"
  op: "Sqrt"
  input: "add_935"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_936/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_936"
  op: "AddV2"
  input: "Sqrt_273"
  input: "add_936/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_276"
  op: "RealDiv"
  input: "add_934"
  input: "add_936"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1494"
  op: "Mul"
  input: "mul_21"
  input: "truediv_276"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1195"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_277"
  op: "Sub"
  input: "ReadVariableOp_1195"
  input: "mul_1494"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_819"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_277"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1196"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_819"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_820"
  op: "AssignVariableOp"
  input: "Mul_1490/ReadVariableOp/Enter"
  input: "add_934"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1197"
  op: "ReadVariableOp"
  input: "Mul_1490/ReadVariableOp/Enter"
  input: "^AssignVariableOp_820"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_821"
  op: "AssignVariableOp"
  input: "Mul_1492/ReadVariableOp/Enter"
  input: "add_935"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1198"
  op: "ReadVariableOp"
  input: "Mul_1492/ReadVariableOp/Enter"
  input: "^AssignVariableOp_821"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1495/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1495/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1495/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1495/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1495"
  op: "Mul"
  input: "Mul_1495/x"
  input: "Mul_1495/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1496/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1496"
  op: "Mul"
  input: "Mul_1496/x"
  input: "CrossReplicaSum_274"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_937"
  op: "AddV2"
  input: "Mul_1495"
  input: "Mul_1496"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1497/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1497/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1497/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1497/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1497"
  op: "Mul"
  input: "Mul_1497/x"
  input: "Mul_1497/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_274"
  op: "Square"
  input: "CrossReplicaSum_274"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1498/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1498"
  op: "Mul"
  input: "Mul_1498/x"
  input: "Square_274"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_938"
  op: "AddV2"
  input: "Mul_1497"
  input: "Mul_1498"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_274"
  op: "Sqrt"
  input: "add_938"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_939/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_939"
  op: "AddV2"
  input: "Sqrt_274"
  input: "add_939/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_277"
  op: "RealDiv"
  input: "add_937"
  input: "add_939"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1199"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1499/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1499"
  op: "Mul"
  input: "mul_1499/x"
  input: "ReadVariableOp_1199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_940"
  op: "AddV2"
  input: "truediv_277"
  input: "mul_1499"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1500"
  op: "Mul"
  input: "mul_21"
  input: "add_940"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1200"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_278"
  op: "Sub"
  input: "ReadVariableOp_1200"
  input: "mul_1500"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_822"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_278"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1201"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_822"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_823"
  op: "AssignVariableOp"
  input: "Mul_1495/ReadVariableOp/Enter"
  input: "add_937"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1202"
  op: "ReadVariableOp"
  input: "Mul_1495/ReadVariableOp/Enter"
  input: "^AssignVariableOp_823"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_824"
  op: "AssignVariableOp"
  input: "Mul_1497/ReadVariableOp/Enter"
  input: "add_938"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1203"
  op: "ReadVariableOp"
  input: "Mul_1497/ReadVariableOp/Enter"
  input: "^AssignVariableOp_824"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1501/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1501/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1501/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1501/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1501"
  op: "Mul"
  input: "Mul_1501/x"
  input: "Mul_1501/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1502/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1502"
  op: "Mul"
  input: "Mul_1502/x"
  input: "CrossReplicaSum_275"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_941"
  op: "AddV2"
  input: "Mul_1501"
  input: "Mul_1502"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1503/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1503/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1503/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1503/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1503"
  op: "Mul"
  input: "Mul_1503/x"
  input: "Mul_1503/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_275"
  op: "Square"
  input: "CrossReplicaSum_275"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1504/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1504"
  op: "Mul"
  input: "Mul_1504/x"
  input: "Square_275"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_942"
  op: "AddV2"
  input: "Mul_1503"
  input: "Mul_1504"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_275"
  op: "Sqrt"
  input: "add_942"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_943/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_943"
  op: "AddV2"
  input: "Sqrt_275"
  input: "add_943/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_278"
  op: "RealDiv"
  input: "add_941"
  input: "add_943"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1505"
  op: "Mul"
  input: "mul_21"
  input: "truediv_278"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1204"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_279"
  op: "Sub"
  input: "ReadVariableOp_1204"
  input: "mul_1505"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_825"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_279"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1205"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_825"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_826"
  op: "AssignVariableOp"
  input: "Mul_1501/ReadVariableOp/Enter"
  input: "add_941"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1206"
  op: "ReadVariableOp"
  input: "Mul_1501/ReadVariableOp/Enter"
  input: "^AssignVariableOp_826"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_827"
  op: "AssignVariableOp"
  input: "Mul_1503/ReadVariableOp/Enter"
  input: "add_942"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1207"
  op: "ReadVariableOp"
  input: "Mul_1503/ReadVariableOp/Enter"
  input: "^AssignVariableOp_827"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1506/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1506/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1506/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1506/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1506"
  op: "Mul"
  input: "Mul_1506/x"
  input: "Mul_1506/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1507/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1507"
  op: "Mul"
  input: "Mul_1507/x"
  input: "CrossReplicaSum_276"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_944"
  op: "AddV2"
  input: "Mul_1506"
  input: "Mul_1507"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1508/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1508/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1508/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1508/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1508"
  op: "Mul"
  input: "Mul_1508/x"
  input: "Mul_1508/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_276"
  op: "Square"
  input: "CrossReplicaSum_276"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1509/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1509"
  op: "Mul"
  input: "Mul_1509/x"
  input: "Square_276"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_945"
  op: "AddV2"
  input: "Mul_1508"
  input: "Mul_1509"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_276"
  op: "Sqrt"
  input: "add_945"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_946/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_946"
  op: "AddV2"
  input: "Sqrt_276"
  input: "add_946/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_279"
  op: "RealDiv"
  input: "add_944"
  input: "add_946"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1208"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1510/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1510"
  op: "Mul"
  input: "mul_1510/x"
  input: "ReadVariableOp_1208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_947"
  op: "AddV2"
  input: "truediv_279"
  input: "mul_1510"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1511"
  op: "Mul"
  input: "mul_21"
  input: "add_947"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1209"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_280"
  op: "Sub"
  input: "ReadVariableOp_1209"
  input: "mul_1511"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_828"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_280"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1210"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_828"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_829"
  op: "AssignVariableOp"
  input: "Mul_1506/ReadVariableOp/Enter"
  input: "add_944"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1211"
  op: "ReadVariableOp"
  input: "Mul_1506/ReadVariableOp/Enter"
  input: "^AssignVariableOp_829"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_830"
  op: "AssignVariableOp"
  input: "Mul_1508/ReadVariableOp/Enter"
  input: "add_945"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1212"
  op: "ReadVariableOp"
  input: "Mul_1508/ReadVariableOp/Enter"
  input: "^AssignVariableOp_830"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1512/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1512/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1512/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1512/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1512"
  op: "Mul"
  input: "Mul_1512/x"
  input: "Mul_1512/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1513/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1513"
  op: "Mul"
  input: "Mul_1513/x"
  input: "CrossReplicaSum_277"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_948"
  op: "AddV2"
  input: "Mul_1512"
  input: "Mul_1513"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1514/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1514/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1514/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1514/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1514"
  op: "Mul"
  input: "Mul_1514/x"
  input: "Mul_1514/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_277"
  op: "Square"
  input: "CrossReplicaSum_277"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1515/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1515"
  op: "Mul"
  input: "Mul_1515/x"
  input: "Square_277"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_949"
  op: "AddV2"
  input: "Mul_1514"
  input: "Mul_1515"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_277"
  op: "Sqrt"
  input: "add_949"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_950/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_950"
  op: "AddV2"
  input: "Sqrt_277"
  input: "add_950/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_280"
  op: "RealDiv"
  input: "add_948"
  input: "add_950"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1516"
  op: "Mul"
  input: "mul_21"
  input: "truediv_280"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1213"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_281"
  op: "Sub"
  input: "ReadVariableOp_1213"
  input: "mul_1516"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_831"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_281"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1214"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_831"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_832"
  op: "AssignVariableOp"
  input: "Mul_1512/ReadVariableOp/Enter"
  input: "add_948"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1215"
  op: "ReadVariableOp"
  input: "Mul_1512/ReadVariableOp/Enter"
  input: "^AssignVariableOp_832"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_833"
  op: "AssignVariableOp"
  input: "Mul_1514/ReadVariableOp/Enter"
  input: "add_949"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1216"
  op: "ReadVariableOp"
  input: "Mul_1514/ReadVariableOp/Enter"
  input: "^AssignVariableOp_833"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1517/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1517/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1517/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1517/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1517"
  op: "Mul"
  input: "Mul_1517/x"
  input: "Mul_1517/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1518/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1518"
  op: "Mul"
  input: "Mul_1518/x"
  input: "CrossReplicaSum_278"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_951"
  op: "AddV2"
  input: "Mul_1517"
  input: "Mul_1518"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1519/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1519/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1519/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1519/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1519"
  op: "Mul"
  input: "Mul_1519/x"
  input: "Mul_1519/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_278"
  op: "Square"
  input: "CrossReplicaSum_278"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1520/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1520"
  op: "Mul"
  input: "Mul_1520/x"
  input: "Square_278"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_952"
  op: "AddV2"
  input: "Mul_1519"
  input: "Mul_1520"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_278"
  op: "Sqrt"
  input: "add_952"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_953/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_953"
  op: "AddV2"
  input: "Sqrt_278"
  input: "add_953/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_281"
  op: "RealDiv"
  input: "add_951"
  input: "add_953"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1521"
  op: "Mul"
  input: "mul_21"
  input: "truediv_281"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1217"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_282"
  op: "Sub"
  input: "ReadVariableOp_1217"
  input: "mul_1521"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_834"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_282"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1218"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_834"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_835"
  op: "AssignVariableOp"
  input: "Mul_1517/ReadVariableOp/Enter"
  input: "add_951"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1219"
  op: "ReadVariableOp"
  input: "Mul_1517/ReadVariableOp/Enter"
  input: "^AssignVariableOp_835"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_836"
  op: "AssignVariableOp"
  input: "Mul_1519/ReadVariableOp/Enter"
  input: "add_952"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1220"
  op: "ReadVariableOp"
  input: "Mul_1519/ReadVariableOp/Enter"
  input: "^AssignVariableOp_836"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1522/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1522/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1522/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1522/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1522"
  op: "Mul"
  input: "Mul_1522/x"
  input: "Mul_1522/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1523/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1523"
  op: "Mul"
  input: "Mul_1523/x"
  input: "CrossReplicaSum_279"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_954"
  op: "AddV2"
  input: "Mul_1522"
  input: "Mul_1523"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1524/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1524/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1524/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1524/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1524"
  op: "Mul"
  input: "Mul_1524/x"
  input: "Mul_1524/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_279"
  op: "Square"
  input: "CrossReplicaSum_279"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1525/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1525"
  op: "Mul"
  input: "Mul_1525/x"
  input: "Square_279"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_955"
  op: "AddV2"
  input: "Mul_1524"
  input: "Mul_1525"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_279"
  op: "Sqrt"
  input: "add_955"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_956/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_956"
  op: "AddV2"
  input: "Sqrt_279"
  input: "add_956/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_282"
  op: "RealDiv"
  input: "add_954"
  input: "add_956"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1526"
  op: "Mul"
  input: "mul_21"
  input: "truediv_282"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1221"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_283"
  op: "Sub"
  input: "ReadVariableOp_1221"
  input: "mul_1526"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_837"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_283"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1222"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_837"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_838"
  op: "AssignVariableOp"
  input: "Mul_1522/ReadVariableOp/Enter"
  input: "add_954"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1223"
  op: "ReadVariableOp"
  input: "Mul_1522/ReadVariableOp/Enter"
  input: "^AssignVariableOp_838"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_839"
  op: "AssignVariableOp"
  input: "Mul_1524/ReadVariableOp/Enter"
  input: "add_955"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1224"
  op: "ReadVariableOp"
  input: "Mul_1524/ReadVariableOp/Enter"
  input: "^AssignVariableOp_839"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1527/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1527/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1527/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1527/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1527"
  op: "Mul"
  input: "Mul_1527/x"
  input: "Mul_1527/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1528/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1528"
  op: "Mul"
  input: "Mul_1528/x"
  input: "CrossReplicaSum_280"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_957"
  op: "AddV2"
  input: "Mul_1527"
  input: "Mul_1528"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1529/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1529/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1529/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1529/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1529"
  op: "Mul"
  input: "Mul_1529/x"
  input: "Mul_1529/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_280"
  op: "Square"
  input: "CrossReplicaSum_280"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1530/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1530"
  op: "Mul"
  input: "Mul_1530/x"
  input: "Square_280"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_958"
  op: "AddV2"
  input: "Mul_1529"
  input: "Mul_1530"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_280"
  op: "Sqrt"
  input: "add_958"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_959/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_959"
  op: "AddV2"
  input: "Sqrt_280"
  input: "add_959/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_283"
  op: "RealDiv"
  input: "add_957"
  input: "add_959"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1225"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1531/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1531"
  op: "Mul"
  input: "mul_1531/x"
  input: "ReadVariableOp_1225"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_960"
  op: "AddV2"
  input: "truediv_283"
  input: "mul_1531"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1532"
  op: "Mul"
  input: "mul_21"
  input: "add_960"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1226"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_284"
  op: "Sub"
  input: "ReadVariableOp_1226"
  input: "mul_1532"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_840"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_284"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1227"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_840"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_841"
  op: "AssignVariableOp"
  input: "Mul_1527/ReadVariableOp/Enter"
  input: "add_957"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1228"
  op: "ReadVariableOp"
  input: "Mul_1527/ReadVariableOp/Enter"
  input: "^AssignVariableOp_841"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_842"
  op: "AssignVariableOp"
  input: "Mul_1529/ReadVariableOp/Enter"
  input: "add_958"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1229"
  op: "ReadVariableOp"
  input: "Mul_1529/ReadVariableOp/Enter"
  input: "^AssignVariableOp_842"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1533/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1533/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1533/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1533/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1533"
  op: "Mul"
  input: "Mul_1533/x"
  input: "Mul_1533/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1534/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1534"
  op: "Mul"
  input: "Mul_1534/x"
  input: "CrossReplicaSum_281"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_961"
  op: "AddV2"
  input: "Mul_1533"
  input: "Mul_1534"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1535/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1535/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1535/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1535/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1535"
  op: "Mul"
  input: "Mul_1535/x"
  input: "Mul_1535/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_281"
  op: "Square"
  input: "CrossReplicaSum_281"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1536/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1536"
  op: "Mul"
  input: "Mul_1536/x"
  input: "Square_281"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_962"
  op: "AddV2"
  input: "Mul_1535"
  input: "Mul_1536"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_281"
  op: "Sqrt"
  input: "add_962"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_963/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_963"
  op: "AddV2"
  input: "Sqrt_281"
  input: "add_963/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_284"
  op: "RealDiv"
  input: "add_961"
  input: "add_963"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1537"
  op: "Mul"
  input: "mul_21"
  input: "truediv_284"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1230"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_285"
  op: "Sub"
  input: "ReadVariableOp_1230"
  input: "mul_1537"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_843"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_285"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1231"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_843"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_844"
  op: "AssignVariableOp"
  input: "Mul_1533/ReadVariableOp/Enter"
  input: "add_961"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1232"
  op: "ReadVariableOp"
  input: "Mul_1533/ReadVariableOp/Enter"
  input: "^AssignVariableOp_844"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_845"
  op: "AssignVariableOp"
  input: "Mul_1535/ReadVariableOp/Enter"
  input: "add_962"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1233"
  op: "ReadVariableOp"
  input: "Mul_1535/ReadVariableOp/Enter"
  input: "^AssignVariableOp_845"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1538/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1538/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1538/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1538/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1538"
  op: "Mul"
  input: "Mul_1538/x"
  input: "Mul_1538/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1539/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1539"
  op: "Mul"
  input: "Mul_1539/x"
  input: "CrossReplicaSum_282"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_964"
  op: "AddV2"
  input: "Mul_1538"
  input: "Mul_1539"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1540/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1540/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1540/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1540/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1540"
  op: "Mul"
  input: "Mul_1540/x"
  input: "Mul_1540/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_282"
  op: "Square"
  input: "CrossReplicaSum_282"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1541/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1541"
  op: "Mul"
  input: "Mul_1541/x"
  input: "Square_282"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_965"
  op: "AddV2"
  input: "Mul_1540"
  input: "Mul_1541"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_282"
  op: "Sqrt"
  input: "add_965"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_966/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_966"
  op: "AddV2"
  input: "Sqrt_282"
  input: "add_966/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_285"
  op: "RealDiv"
  input: "add_964"
  input: "add_966"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1234"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1542/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1542"
  op: "Mul"
  input: "mul_1542/x"
  input: "ReadVariableOp_1234"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_967"
  op: "AddV2"
  input: "truediv_285"
  input: "mul_1542"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1543"
  op: "Mul"
  input: "mul_21"
  input: "add_967"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1235"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_286"
  op: "Sub"
  input: "ReadVariableOp_1235"
  input: "mul_1543"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_846"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_286"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1236"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_846"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_847"
  op: "AssignVariableOp"
  input: "Mul_1538/ReadVariableOp/Enter"
  input: "add_964"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1237"
  op: "ReadVariableOp"
  input: "Mul_1538/ReadVariableOp/Enter"
  input: "^AssignVariableOp_847"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_848"
  op: "AssignVariableOp"
  input: "Mul_1540/ReadVariableOp/Enter"
  input: "add_965"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1238"
  op: "ReadVariableOp"
  input: "Mul_1540/ReadVariableOp/Enter"
  input: "^AssignVariableOp_848"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_m"
  input: "electra/encoder/layer_4/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_v"
  input: "electra/encoder/layer_4/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1544/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1544/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1544/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1544/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1544"
  op: "Mul"
  input: "Mul_1544/x"
  input: "Mul_1544/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1545/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1545"
  op: "Mul"
  input: "Mul_1545/x"
  input: "CrossReplicaSum_283"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_968"
  op: "AddV2"
  input: "Mul_1544"
  input: "Mul_1545"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1546/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1546/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1546/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1546/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1546"
  op: "Mul"
  input: "Mul_1546/x"
  input: "Mul_1546/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_283"
  op: "Square"
  input: "CrossReplicaSum_283"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1547/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1547"
  op: "Mul"
  input: "Mul_1547/x"
  input: "Square_283"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_969"
  op: "AddV2"
  input: "Mul_1546"
  input: "Mul_1547"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_283"
  op: "Sqrt"
  input: "add_969"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_970/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_970"
  op: "AddV2"
  input: "Sqrt_283"
  input: "add_970/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_286"
  op: "RealDiv"
  input: "add_968"
  input: "add_970"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1548"
  op: "Mul"
  input: "mul_21"
  input: "truediv_286"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1239"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_287"
  op: "Sub"
  input: "ReadVariableOp_1239"
  input: "mul_1548"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_849"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_287"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1240"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_849"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_850"
  op: "AssignVariableOp"
  input: "Mul_1544/ReadVariableOp/Enter"
  input: "add_968"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1241"
  op: "ReadVariableOp"
  input: "Mul_1544/ReadVariableOp/Enter"
  input: "^AssignVariableOp_850"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_851"
  op: "AssignVariableOp"
  input: "Mul_1546/ReadVariableOp/Enter"
  input: "add_969"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1242"
  op: "ReadVariableOp"
  input: "Mul_1546/ReadVariableOp/Enter"
  input: "^AssignVariableOp_851"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1549/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1549/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1549/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1549/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1549"
  op: "Mul"
  input: "Mul_1549/x"
  input: "Mul_1549/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1550/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1550"
  op: "Mul"
  input: "Mul_1550/x"
  input: "CrossReplicaSum_284"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_971"
  op: "AddV2"
  input: "Mul_1549"
  input: "Mul_1550"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1551/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1551/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1551/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1551/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1551"
  op: "Mul"
  input: "Mul_1551/x"
  input: "Mul_1551/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_284"
  op: "Square"
  input: "CrossReplicaSum_284"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1552/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1552"
  op: "Mul"
  input: "Mul_1552/x"
  input: "Square_284"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_972"
  op: "AddV2"
  input: "Mul_1551"
  input: "Mul_1552"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_284"
  op: "Sqrt"
  input: "add_972"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_973/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_973"
  op: "AddV2"
  input: "Sqrt_284"
  input: "add_973/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_287"
  op: "RealDiv"
  input: "add_971"
  input: "add_973"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1553"
  op: "Mul"
  input: "mul_21"
  input: "truediv_287"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1243"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_288"
  op: "Sub"
  input: "ReadVariableOp_1243"
  input: "mul_1553"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_852"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_288"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1244"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_852"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_853"
  op: "AssignVariableOp"
  input: "Mul_1549/ReadVariableOp/Enter"
  input: "add_971"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1245"
  op: "ReadVariableOp"
  input: "Mul_1549/ReadVariableOp/Enter"
  input: "^AssignVariableOp_853"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_854"
  op: "AssignVariableOp"
  input: "Mul_1551/ReadVariableOp/Enter"
  input: "add_972"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1246"
  op: "ReadVariableOp"
  input: "Mul_1551/ReadVariableOp/Enter"
  input: "^AssignVariableOp_854"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1554/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1554/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1554/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1554/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1554"
  op: "Mul"
  input: "Mul_1554/x"
  input: "Mul_1554/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1555/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1555"
  op: "Mul"
  input: "Mul_1555/x"
  input: "CrossReplicaSum_285"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_974"
  op: "AddV2"
  input: "Mul_1554"
  input: "Mul_1555"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1556/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1556/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1556/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1556/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1556"
  op: "Mul"
  input: "Mul_1556/x"
  input: "Mul_1556/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_285"
  op: "Square"
  input: "CrossReplicaSum_285"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1557/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1557"
  op: "Mul"
  input: "Mul_1557/x"
  input: "Square_285"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_975"
  op: "AddV2"
  input: "Mul_1556"
  input: "Mul_1557"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_285"
  op: "Sqrt"
  input: "add_975"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_976/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_976"
  op: "AddV2"
  input: "Sqrt_285"
  input: "add_976/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_288"
  op: "RealDiv"
  input: "add_974"
  input: "add_976"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1558"
  op: "Mul"
  input: "mul_21"
  input: "truediv_288"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1247"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_289"
  op: "Sub"
  input: "ReadVariableOp_1247"
  input: "mul_1558"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_855"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_289"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1248"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_855"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_856"
  op: "AssignVariableOp"
  input: "Mul_1554/ReadVariableOp/Enter"
  input: "add_974"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1249"
  op: "ReadVariableOp"
  input: "Mul_1554/ReadVariableOp/Enter"
  input: "^AssignVariableOp_856"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_857"
  op: "AssignVariableOp"
  input: "Mul_1556/ReadVariableOp/Enter"
  input: "add_975"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1250"
  op: "ReadVariableOp"
  input: "Mul_1556/ReadVariableOp/Enter"
  input: "^AssignVariableOp_857"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1559/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1559/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1559/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1559/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1559"
  op: "Mul"
  input: "Mul_1559/x"
  input: "Mul_1559/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1560/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1560"
  op: "Mul"
  input: "Mul_1560/x"
  input: "CrossReplicaSum_286"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_977"
  op: "AddV2"
  input: "Mul_1559"
  input: "Mul_1560"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1561/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1561/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1561/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1561/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1561"
  op: "Mul"
  input: "Mul_1561/x"
  input: "Mul_1561/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_286"
  op: "Square"
  input: "CrossReplicaSum_286"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1562/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1562"
  op: "Mul"
  input: "Mul_1562/x"
  input: "Square_286"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_978"
  op: "AddV2"
  input: "Mul_1561"
  input: "Mul_1562"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_286"
  op: "Sqrt"
  input: "add_978"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_979/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_979"
  op: "AddV2"
  input: "Sqrt_286"
  input: "add_979/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_289"
  op: "RealDiv"
  input: "add_977"
  input: "add_979"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1251"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1563/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1563"
  op: "Mul"
  input: "mul_1563/x"
  input: "ReadVariableOp_1251"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_980"
  op: "AddV2"
  input: "truediv_289"
  input: "mul_1563"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1564"
  op: "Mul"
  input: "mul_21"
  input: "add_980"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1252"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_290"
  op: "Sub"
  input: "ReadVariableOp_1252"
  input: "mul_1564"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_858"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_290"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1253"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_858"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_859"
  op: "AssignVariableOp"
  input: "Mul_1559/ReadVariableOp/Enter"
  input: "add_977"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1254"
  op: "ReadVariableOp"
  input: "Mul_1559/ReadVariableOp/Enter"
  input: "^AssignVariableOp_859"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_860"
  op: "AssignVariableOp"
  input: "Mul_1561/ReadVariableOp/Enter"
  input: "add_978"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1255"
  op: "ReadVariableOp"
  input: "Mul_1561/ReadVariableOp/Enter"
  input: "^AssignVariableOp_860"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1565/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1565/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1565/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1565/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1565"
  op: "Mul"
  input: "Mul_1565/x"
  input: "Mul_1565/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1566/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1566"
  op: "Mul"
  input: "Mul_1566/x"
  input: "CrossReplicaSum_287"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_981"
  op: "AddV2"
  input: "Mul_1565"
  input: "Mul_1566"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1567/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1567/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1567/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1567/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1567"
  op: "Mul"
  input: "Mul_1567/x"
  input: "Mul_1567/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_287"
  op: "Square"
  input: "CrossReplicaSum_287"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1568/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1568"
  op: "Mul"
  input: "Mul_1568/x"
  input: "Square_287"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_982"
  op: "AddV2"
  input: "Mul_1567"
  input: "Mul_1568"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_287"
  op: "Sqrt"
  input: "add_982"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_983/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_983"
  op: "AddV2"
  input: "Sqrt_287"
  input: "add_983/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_290"
  op: "RealDiv"
  input: "add_981"
  input: "add_983"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1569"
  op: "Mul"
  input: "mul_21"
  input: "truediv_290"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1256"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_291"
  op: "Sub"
  input: "ReadVariableOp_1256"
  input: "mul_1569"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_861"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_291"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1257"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_861"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_862"
  op: "AssignVariableOp"
  input: "Mul_1565/ReadVariableOp/Enter"
  input: "add_981"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1258"
  op: "ReadVariableOp"
  input: "Mul_1565/ReadVariableOp/Enter"
  input: "^AssignVariableOp_862"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_863"
  op: "AssignVariableOp"
  input: "Mul_1567/ReadVariableOp/Enter"
  input: "add_982"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1259"
  op: "ReadVariableOp"
  input: "Mul_1567/ReadVariableOp/Enter"
  input: "^AssignVariableOp_863"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1570/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1570/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1570/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1570/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1570"
  op: "Mul"
  input: "Mul_1570/x"
  input: "Mul_1570/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1571/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1571"
  op: "Mul"
  input: "Mul_1571/x"
  input: "CrossReplicaSum_288"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_984"
  op: "AddV2"
  input: "Mul_1570"
  input: "Mul_1571"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1572/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1572/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1572/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1572/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1572"
  op: "Mul"
  input: "Mul_1572/x"
  input: "Mul_1572/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_288"
  op: "Square"
  input: "CrossReplicaSum_288"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1573/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1573"
  op: "Mul"
  input: "Mul_1573/x"
  input: "Square_288"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_985"
  op: "AddV2"
  input: "Mul_1572"
  input: "Mul_1573"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_288"
  op: "Sqrt"
  input: "add_985"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_986/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_986"
  op: "AddV2"
  input: "Sqrt_288"
  input: "add_986/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_291"
  op: "RealDiv"
  input: "add_984"
  input: "add_986"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1260"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1574/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1574"
  op: "Mul"
  input: "mul_1574/x"
  input: "ReadVariableOp_1260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_987"
  op: "AddV2"
  input: "truediv_291"
  input: "mul_1574"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1575"
  op: "Mul"
  input: "mul_21"
  input: "add_987"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1261"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_292"
  op: "Sub"
  input: "ReadVariableOp_1261"
  input: "mul_1575"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_864"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_292"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1262"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_864"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_865"
  op: "AssignVariableOp"
  input: "Mul_1570/ReadVariableOp/Enter"
  input: "add_984"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1263"
  op: "ReadVariableOp"
  input: "Mul_1570/ReadVariableOp/Enter"
  input: "^AssignVariableOp_865"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_866"
  op: "AssignVariableOp"
  input: "Mul_1572/ReadVariableOp/Enter"
  input: "add_985"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1264"
  op: "ReadVariableOp"
  input: "Mul_1572/ReadVariableOp/Enter"
  input: "^AssignVariableOp_866"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1576/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1576/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1576/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1576/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1576"
  op: "Mul"
  input: "Mul_1576/x"
  input: "Mul_1576/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1577/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1577"
  op: "Mul"
  input: "Mul_1577/x"
  input: "CrossReplicaSum_289"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_988"
  op: "AddV2"
  input: "Mul_1576"
  input: "Mul_1577"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1578/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1578/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1578/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1578/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1578"
  op: "Mul"
  input: "Mul_1578/x"
  input: "Mul_1578/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_289"
  op: "Square"
  input: "CrossReplicaSum_289"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1579/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1579"
  op: "Mul"
  input: "Mul_1579/x"
  input: "Square_289"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_989"
  op: "AddV2"
  input: "Mul_1578"
  input: "Mul_1579"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_289"
  op: "Sqrt"
  input: "add_989"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_990/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_990"
  op: "AddV2"
  input: "Sqrt_289"
  input: "add_990/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_292"
  op: "RealDiv"
  input: "add_988"
  input: "add_990"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1580"
  op: "Mul"
  input: "mul_21"
  input: "truediv_292"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1265"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_293"
  op: "Sub"
  input: "ReadVariableOp_1265"
  input: "mul_1580"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_867"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_293"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1266"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_867"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_868"
  op: "AssignVariableOp"
  input: "Mul_1576/ReadVariableOp/Enter"
  input: "add_988"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1267"
  op: "ReadVariableOp"
  input: "Mul_1576/ReadVariableOp/Enter"
  input: "^AssignVariableOp_868"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_869"
  op: "AssignVariableOp"
  input: "Mul_1578/ReadVariableOp/Enter"
  input: "add_989"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1268"
  op: "ReadVariableOp"
  input: "Mul_1578/ReadVariableOp/Enter"
  input: "^AssignVariableOp_869"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1581/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1581/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1581/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1581/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1581"
  op: "Mul"
  input: "Mul_1581/x"
  input: "Mul_1581/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1582/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1582"
  op: "Mul"
  input: "Mul_1582/x"
  input: "CrossReplicaSum_290"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_991"
  op: "AddV2"
  input: "Mul_1581"
  input: "Mul_1582"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1583/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1583/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1583/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1583/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1583"
  op: "Mul"
  input: "Mul_1583/x"
  input: "Mul_1583/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_290"
  op: "Square"
  input: "CrossReplicaSum_290"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1584/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1584"
  op: "Mul"
  input: "Mul_1584/x"
  input: "Square_290"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_992"
  op: "AddV2"
  input: "Mul_1583"
  input: "Mul_1584"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_290"
  op: "Sqrt"
  input: "add_992"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_993/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_993"
  op: "AddV2"
  input: "Sqrt_290"
  input: "add_993/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_293"
  op: "RealDiv"
  input: "add_991"
  input: "add_993"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1269"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1585/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1585"
  op: "Mul"
  input: "mul_1585/x"
  input: "ReadVariableOp_1269"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_994"
  op: "AddV2"
  input: "truediv_293"
  input: "mul_1585"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1586"
  op: "Mul"
  input: "mul_21"
  input: "add_994"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1270"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_294"
  op: "Sub"
  input: "ReadVariableOp_1270"
  input: "mul_1586"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_870"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_294"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1271"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_870"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_871"
  op: "AssignVariableOp"
  input: "Mul_1581/ReadVariableOp/Enter"
  input: "add_991"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1272"
  op: "ReadVariableOp"
  input: "Mul_1581/ReadVariableOp/Enter"
  input: "^AssignVariableOp_871"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_872"
  op: "AssignVariableOp"
  input: "Mul_1583/ReadVariableOp/Enter"
  input: "add_992"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1273"
  op: "ReadVariableOp"
  input: "Mul_1583/ReadVariableOp/Enter"
  input: "^AssignVariableOp_872"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1587/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1587/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1587/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1587/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1587"
  op: "Mul"
  input: "Mul_1587/x"
  input: "Mul_1587/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1588/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1588"
  op: "Mul"
  input: "Mul_1588/x"
  input: "CrossReplicaSum_291"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_995"
  op: "AddV2"
  input: "Mul_1587"
  input: "Mul_1588"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1589/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1589/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1589/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1589/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1589"
  op: "Mul"
  input: "Mul_1589/x"
  input: "Mul_1589/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_291"
  op: "Square"
  input: "CrossReplicaSum_291"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1590/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1590"
  op: "Mul"
  input: "Mul_1590/x"
  input: "Square_291"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_996"
  op: "AddV2"
  input: "Mul_1589"
  input: "Mul_1590"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_291"
  op: "Sqrt"
  input: "add_996"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_997/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_997"
  op: "AddV2"
  input: "Sqrt_291"
  input: "add_997/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_294"
  op: "RealDiv"
  input: "add_995"
  input: "add_997"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1591"
  op: "Mul"
  input: "mul_21"
  input: "truediv_294"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1274"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_295"
  op: "Sub"
  input: "ReadVariableOp_1274"
  input: "mul_1591"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_873"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_295"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1275"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_873"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_874"
  op: "AssignVariableOp"
  input: "Mul_1587/ReadVariableOp/Enter"
  input: "add_995"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1276"
  op: "ReadVariableOp"
  input: "Mul_1587/ReadVariableOp/Enter"
  input: "^AssignVariableOp_874"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_875"
  op: "AssignVariableOp"
  input: "Mul_1589/ReadVariableOp/Enter"
  input: "add_996"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1277"
  op: "ReadVariableOp"
  input: "Mul_1589/ReadVariableOp/Enter"
  input: "^AssignVariableOp_875"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1592/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1592/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1592/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1592/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1592"
  op: "Mul"
  input: "Mul_1592/x"
  input: "Mul_1592/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1593/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1593"
  op: "Mul"
  input: "Mul_1593/x"
  input: "CrossReplicaSum_292"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_998"
  op: "AddV2"
  input: "Mul_1592"
  input: "Mul_1593"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1594/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1594/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1594/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1594/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1594"
  op: "Mul"
  input: "Mul_1594/x"
  input: "Mul_1594/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_292"
  op: "Square"
  input: "CrossReplicaSum_292"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1595/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1595"
  op: "Mul"
  input: "Mul_1595/x"
  input: "Square_292"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_999"
  op: "AddV2"
  input: "Mul_1594"
  input: "Mul_1595"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_292"
  op: "Sqrt"
  input: "add_999"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1000/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1000"
  op: "AddV2"
  input: "Sqrt_292"
  input: "add_1000/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_295"
  op: "RealDiv"
  input: "add_998"
  input: "add_1000"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1278"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1596/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1596"
  op: "Mul"
  input: "mul_1596/x"
  input: "ReadVariableOp_1278"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1001"
  op: "AddV2"
  input: "truediv_295"
  input: "mul_1596"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1597"
  op: "Mul"
  input: "mul_21"
  input: "add_1001"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1279"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_296"
  op: "Sub"
  input: "ReadVariableOp_1279"
  input: "mul_1597"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_876"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_296"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1280"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_876"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_877"
  op: "AssignVariableOp"
  input: "Mul_1592/ReadVariableOp/Enter"
  input: "add_998"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1281"
  op: "ReadVariableOp"
  input: "Mul_1592/ReadVariableOp/Enter"
  input: "^AssignVariableOp_877"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_878"
  op: "AssignVariableOp"
  input: "Mul_1594/ReadVariableOp/Enter"
  input: "add_999"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1282"
  op: "ReadVariableOp"
  input: "Mul_1594/ReadVariableOp/Enter"
  input: "^AssignVariableOp_878"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1598/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1598/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1598/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1598/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1598"
  op: "Mul"
  input: "Mul_1598/x"
  input: "Mul_1598/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1599/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1599"
  op: "Mul"
  input: "Mul_1599/x"
  input: "CrossReplicaSum_293"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1002"
  op: "AddV2"
  input: "Mul_1598"
  input: "Mul_1599"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1600/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1600/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1600/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1600/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1600"
  op: "Mul"
  input: "Mul_1600/x"
  input: "Mul_1600/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_293"
  op: "Square"
  input: "CrossReplicaSum_293"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1601/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1601"
  op: "Mul"
  input: "Mul_1601/x"
  input: "Square_293"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1003"
  op: "AddV2"
  input: "Mul_1600"
  input: "Mul_1601"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_293"
  op: "Sqrt"
  input: "add_1003"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1004/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1004"
  op: "AddV2"
  input: "Sqrt_293"
  input: "add_1004/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_296"
  op: "RealDiv"
  input: "add_1002"
  input: "add_1004"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1602"
  op: "Mul"
  input: "mul_21"
  input: "truediv_296"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1283"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_297"
  op: "Sub"
  input: "ReadVariableOp_1283"
  input: "mul_1602"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_879"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_297"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1284"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_879"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_880"
  op: "AssignVariableOp"
  input: "Mul_1598/ReadVariableOp/Enter"
  input: "add_1002"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1285"
  op: "ReadVariableOp"
  input: "Mul_1598/ReadVariableOp/Enter"
  input: "^AssignVariableOp_880"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_881"
  op: "AssignVariableOp"
  input: "Mul_1600/ReadVariableOp/Enter"
  input: "add_1003"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1286"
  op: "ReadVariableOp"
  input: "Mul_1600/ReadVariableOp/Enter"
  input: "^AssignVariableOp_881"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1603/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1603/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1603/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1603/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1603"
  op: "Mul"
  input: "Mul_1603/x"
  input: "Mul_1603/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1604/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1604"
  op: "Mul"
  input: "Mul_1604/x"
  input: "CrossReplicaSum_294"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1005"
  op: "AddV2"
  input: "Mul_1603"
  input: "Mul_1604"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1605/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1605/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1605/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1605/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1605"
  op: "Mul"
  input: "Mul_1605/x"
  input: "Mul_1605/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_294"
  op: "Square"
  input: "CrossReplicaSum_294"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1606/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1606"
  op: "Mul"
  input: "Mul_1606/x"
  input: "Square_294"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1006"
  op: "AddV2"
  input: "Mul_1605"
  input: "Mul_1606"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_294"
  op: "Sqrt"
  input: "add_1006"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1007/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1007"
  op: "AddV2"
  input: "Sqrt_294"
  input: "add_1007/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_297"
  op: "RealDiv"
  input: "add_1005"
  input: "add_1007"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1607"
  op: "Mul"
  input: "mul_21"
  input: "truediv_297"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1287"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_298"
  op: "Sub"
  input: "ReadVariableOp_1287"
  input: "mul_1607"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_882"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_298"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1288"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_882"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_883"
  op: "AssignVariableOp"
  input: "Mul_1603/ReadVariableOp/Enter"
  input: "add_1005"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1289"
  op: "ReadVariableOp"
  input: "Mul_1603/ReadVariableOp/Enter"
  input: "^AssignVariableOp_883"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_884"
  op: "AssignVariableOp"
  input: "Mul_1605/ReadVariableOp/Enter"
  input: "add_1006"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1290"
  op: "ReadVariableOp"
  input: "Mul_1605/ReadVariableOp/Enter"
  input: "^AssignVariableOp_884"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1608/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1608/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1608/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1608/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1608"
  op: "Mul"
  input: "Mul_1608/x"
  input: "Mul_1608/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1609/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1609"
  op: "Mul"
  input: "Mul_1609/x"
  input: "CrossReplicaSum_295"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1008"
  op: "AddV2"
  input: "Mul_1608"
  input: "Mul_1609"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1610/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1610/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1610/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1610/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1610"
  op: "Mul"
  input: "Mul_1610/x"
  input: "Mul_1610/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_295"
  op: "Square"
  input: "CrossReplicaSum_295"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1611/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1611"
  op: "Mul"
  input: "Mul_1611/x"
  input: "Square_295"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1009"
  op: "AddV2"
  input: "Mul_1610"
  input: "Mul_1611"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_295"
  op: "Sqrt"
  input: "add_1009"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1010/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1010"
  op: "AddV2"
  input: "Sqrt_295"
  input: "add_1010/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_298"
  op: "RealDiv"
  input: "add_1008"
  input: "add_1010"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1612"
  op: "Mul"
  input: "mul_21"
  input: "truediv_298"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1291"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_299"
  op: "Sub"
  input: "ReadVariableOp_1291"
  input: "mul_1612"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_885"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_299"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1292"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_885"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_886"
  op: "AssignVariableOp"
  input: "Mul_1608/ReadVariableOp/Enter"
  input: "add_1008"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1293"
  op: "ReadVariableOp"
  input: "Mul_1608/ReadVariableOp/Enter"
  input: "^AssignVariableOp_886"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_887"
  op: "AssignVariableOp"
  input: "Mul_1610/ReadVariableOp/Enter"
  input: "add_1009"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1294"
  op: "ReadVariableOp"
  input: "Mul_1610/ReadVariableOp/Enter"
  input: "^AssignVariableOp_887"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1613/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1613/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1613/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1613/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1613"
  op: "Mul"
  input: "Mul_1613/x"
  input: "Mul_1613/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1614/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1614"
  op: "Mul"
  input: "Mul_1614/x"
  input: "CrossReplicaSum_296"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1011"
  op: "AddV2"
  input: "Mul_1613"
  input: "Mul_1614"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1615/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1615/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1615/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1615/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1615"
  op: "Mul"
  input: "Mul_1615/x"
  input: "Mul_1615/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_296"
  op: "Square"
  input: "CrossReplicaSum_296"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1616/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1616"
  op: "Mul"
  input: "Mul_1616/x"
  input: "Square_296"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1012"
  op: "AddV2"
  input: "Mul_1615"
  input: "Mul_1616"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_296"
  op: "Sqrt"
  input: "add_1012"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1013/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1013"
  op: "AddV2"
  input: "Sqrt_296"
  input: "add_1013/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_299"
  op: "RealDiv"
  input: "add_1011"
  input: "add_1013"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1295"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1617/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1617"
  op: "Mul"
  input: "mul_1617/x"
  input: "ReadVariableOp_1295"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1014"
  op: "AddV2"
  input: "truediv_299"
  input: "mul_1617"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1618"
  op: "Mul"
  input: "mul_21"
  input: "add_1014"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1296"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_300"
  op: "Sub"
  input: "ReadVariableOp_1296"
  input: "mul_1618"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_888"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_300"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1297"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_888"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_889"
  op: "AssignVariableOp"
  input: "Mul_1613/ReadVariableOp/Enter"
  input: "add_1011"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1298"
  op: "ReadVariableOp"
  input: "Mul_1613/ReadVariableOp/Enter"
  input: "^AssignVariableOp_889"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_890"
  op: "AssignVariableOp"
  input: "Mul_1615/ReadVariableOp/Enter"
  input: "add_1012"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1299"
  op: "ReadVariableOp"
  input: "Mul_1615/ReadVariableOp/Enter"
  input: "^AssignVariableOp_890"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1619/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1619/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1619/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1619/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1619"
  op: "Mul"
  input: "Mul_1619/x"
  input: "Mul_1619/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1620/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1620"
  op: "Mul"
  input: "Mul_1620/x"
  input: "CrossReplicaSum_297"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1015"
  op: "AddV2"
  input: "Mul_1619"
  input: "Mul_1620"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1621/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1621/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1621/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1621/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1621"
  op: "Mul"
  input: "Mul_1621/x"
  input: "Mul_1621/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_297"
  op: "Square"
  input: "CrossReplicaSum_297"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1622/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1622"
  op: "Mul"
  input: "Mul_1622/x"
  input: "Square_297"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1016"
  op: "AddV2"
  input: "Mul_1621"
  input: "Mul_1622"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_297"
  op: "Sqrt"
  input: "add_1016"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1017/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1017"
  op: "AddV2"
  input: "Sqrt_297"
  input: "add_1017/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_300"
  op: "RealDiv"
  input: "add_1015"
  input: "add_1017"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1623"
  op: "Mul"
  input: "mul_21"
  input: "truediv_300"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1300"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_301"
  op: "Sub"
  input: "ReadVariableOp_1300"
  input: "mul_1623"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_891"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_301"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1301"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_891"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_892"
  op: "AssignVariableOp"
  input: "Mul_1619/ReadVariableOp/Enter"
  input: "add_1015"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1302"
  op: "ReadVariableOp"
  input: "Mul_1619/ReadVariableOp/Enter"
  input: "^AssignVariableOp_892"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_893"
  op: "AssignVariableOp"
  input: "Mul_1621/ReadVariableOp/Enter"
  input: "add_1016"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1303"
  op: "ReadVariableOp"
  input: "Mul_1621/ReadVariableOp/Enter"
  input: "^AssignVariableOp_893"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1624/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1624/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1624/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1624/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1624"
  op: "Mul"
  input: "Mul_1624/x"
  input: "Mul_1624/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1625/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1625"
  op: "Mul"
  input: "Mul_1625/x"
  input: "CrossReplicaSum_298"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1018"
  op: "AddV2"
  input: "Mul_1624"
  input: "Mul_1625"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1626/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1626/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1626/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1626/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1626"
  op: "Mul"
  input: "Mul_1626/x"
  input: "Mul_1626/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_298"
  op: "Square"
  input: "CrossReplicaSum_298"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1627/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1627"
  op: "Mul"
  input: "Mul_1627/x"
  input: "Square_298"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1019"
  op: "AddV2"
  input: "Mul_1626"
  input: "Mul_1627"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_298"
  op: "Sqrt"
  input: "add_1019"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1020/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1020"
  op: "AddV2"
  input: "Sqrt_298"
  input: "add_1020/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_301"
  op: "RealDiv"
  input: "add_1018"
  input: "add_1020"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1304"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1628/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1628"
  op: "Mul"
  input: "mul_1628/x"
  input: "ReadVariableOp_1304"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1021"
  op: "AddV2"
  input: "truediv_301"
  input: "mul_1628"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1629"
  op: "Mul"
  input: "mul_21"
  input: "add_1021"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1305"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_302"
  op: "Sub"
  input: "ReadVariableOp_1305"
  input: "mul_1629"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_894"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_302"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1306"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_894"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_895"
  op: "AssignVariableOp"
  input: "Mul_1624/ReadVariableOp/Enter"
  input: "add_1018"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1307"
  op: "ReadVariableOp"
  input: "Mul_1624/ReadVariableOp/Enter"
  input: "^AssignVariableOp_895"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_896"
  op: "AssignVariableOp"
  input: "Mul_1626/ReadVariableOp/Enter"
  input: "add_1019"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1308"
  op: "ReadVariableOp"
  input: "Mul_1626/ReadVariableOp/Enter"
  input: "^AssignVariableOp_896"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_m"
  input: "electra/encoder/layer_5/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_v"
  input: "electra/encoder/layer_5/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1630/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1630/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1630/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1630/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1630"
  op: "Mul"
  input: "Mul_1630/x"
  input: "Mul_1630/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1631/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1631"
  op: "Mul"
  input: "Mul_1631/x"
  input: "CrossReplicaSum_299"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1022"
  op: "AddV2"
  input: "Mul_1630"
  input: "Mul_1631"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1632/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1632/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1632/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1632/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1632"
  op: "Mul"
  input: "Mul_1632/x"
  input: "Mul_1632/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_299"
  op: "Square"
  input: "CrossReplicaSum_299"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1633/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1633"
  op: "Mul"
  input: "Mul_1633/x"
  input: "Square_299"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1023"
  op: "AddV2"
  input: "Mul_1632"
  input: "Mul_1633"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_299"
  op: "Sqrt"
  input: "add_1023"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1024/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1024"
  op: "AddV2"
  input: "Sqrt_299"
  input: "add_1024/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_302"
  op: "RealDiv"
  input: "add_1022"
  input: "add_1024"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1634"
  op: "Mul"
  input: "mul_21"
  input: "truediv_302"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1309"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_303"
  op: "Sub"
  input: "ReadVariableOp_1309"
  input: "mul_1634"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_897"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_303"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1310"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_897"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_898"
  op: "AssignVariableOp"
  input: "Mul_1630/ReadVariableOp/Enter"
  input: "add_1022"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1311"
  op: "ReadVariableOp"
  input: "Mul_1630/ReadVariableOp/Enter"
  input: "^AssignVariableOp_898"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_899"
  op: "AssignVariableOp"
  input: "Mul_1632/ReadVariableOp/Enter"
  input: "add_1023"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1312"
  op: "ReadVariableOp"
  input: "Mul_1632/ReadVariableOp/Enter"
  input: "^AssignVariableOp_899"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1635/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1635/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1635/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1635/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1635"
  op: "Mul"
  input: "Mul_1635/x"
  input: "Mul_1635/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1636/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1636"
  op: "Mul"
  input: "Mul_1636/x"
  input: "CrossReplicaSum_300"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1025"
  op: "AddV2"
  input: "Mul_1635"
  input: "Mul_1636"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1637/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1637/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1637/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1637/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1637"
  op: "Mul"
  input: "Mul_1637/x"
  input: "Mul_1637/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_300"
  op: "Square"
  input: "CrossReplicaSum_300"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1638/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1638"
  op: "Mul"
  input: "Mul_1638/x"
  input: "Square_300"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1026"
  op: "AddV2"
  input: "Mul_1637"
  input: "Mul_1638"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_300"
  op: "Sqrt"
  input: "add_1026"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1027/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1027"
  op: "AddV2"
  input: "Sqrt_300"
  input: "add_1027/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_303"
  op: "RealDiv"
  input: "add_1025"
  input: "add_1027"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1639"
  op: "Mul"
  input: "mul_21"
  input: "truediv_303"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1313"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_304"
  op: "Sub"
  input: "ReadVariableOp_1313"
  input: "mul_1639"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_900"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_304"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1314"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_900"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_901"
  op: "AssignVariableOp"
  input: "Mul_1635/ReadVariableOp/Enter"
  input: "add_1025"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1315"
  op: "ReadVariableOp"
  input: "Mul_1635/ReadVariableOp/Enter"
  input: "^AssignVariableOp_901"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_902"
  op: "AssignVariableOp"
  input: "Mul_1637/ReadVariableOp/Enter"
  input: "add_1026"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1316"
  op: "ReadVariableOp"
  input: "Mul_1637/ReadVariableOp/Enter"
  input: "^AssignVariableOp_902"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1640/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1640/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1640/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1640/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1640"
  op: "Mul"
  input: "Mul_1640/x"
  input: "Mul_1640/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1641/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1641"
  op: "Mul"
  input: "Mul_1641/x"
  input: "CrossReplicaSum_301"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1028"
  op: "AddV2"
  input: "Mul_1640"
  input: "Mul_1641"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1642/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1642/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1642/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1642/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1642"
  op: "Mul"
  input: "Mul_1642/x"
  input: "Mul_1642/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_301"
  op: "Square"
  input: "CrossReplicaSum_301"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1643/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1643"
  op: "Mul"
  input: "Mul_1643/x"
  input: "Square_301"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1029"
  op: "AddV2"
  input: "Mul_1642"
  input: "Mul_1643"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_301"
  op: "Sqrt"
  input: "add_1029"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1030/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1030"
  op: "AddV2"
  input: "Sqrt_301"
  input: "add_1030/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_304"
  op: "RealDiv"
  input: "add_1028"
  input: "add_1030"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1644"
  op: "Mul"
  input: "mul_21"
  input: "truediv_304"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1317"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_305"
  op: "Sub"
  input: "ReadVariableOp_1317"
  input: "mul_1644"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_903"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_305"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1318"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_903"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_904"
  op: "AssignVariableOp"
  input: "Mul_1640/ReadVariableOp/Enter"
  input: "add_1028"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1319"
  op: "ReadVariableOp"
  input: "Mul_1640/ReadVariableOp/Enter"
  input: "^AssignVariableOp_904"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_905"
  op: "AssignVariableOp"
  input: "Mul_1642/ReadVariableOp/Enter"
  input: "add_1029"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1320"
  op: "ReadVariableOp"
  input: "Mul_1642/ReadVariableOp/Enter"
  input: "^AssignVariableOp_905"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1645/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1645/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1645/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1645/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1645"
  op: "Mul"
  input: "Mul_1645/x"
  input: "Mul_1645/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1646/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1646"
  op: "Mul"
  input: "Mul_1646/x"
  input: "CrossReplicaSum_302"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1031"
  op: "AddV2"
  input: "Mul_1645"
  input: "Mul_1646"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1647/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1647/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1647/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1647/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1647"
  op: "Mul"
  input: "Mul_1647/x"
  input: "Mul_1647/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_302"
  op: "Square"
  input: "CrossReplicaSum_302"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1648/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1648"
  op: "Mul"
  input: "Mul_1648/x"
  input: "Square_302"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1032"
  op: "AddV2"
  input: "Mul_1647"
  input: "Mul_1648"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_302"
  op: "Sqrt"
  input: "add_1032"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1033/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1033"
  op: "AddV2"
  input: "Sqrt_302"
  input: "add_1033/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_305"
  op: "RealDiv"
  input: "add_1031"
  input: "add_1033"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1321"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1649/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1649"
  op: "Mul"
  input: "mul_1649/x"
  input: "ReadVariableOp_1321"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1034"
  op: "AddV2"
  input: "truediv_305"
  input: "mul_1649"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1650"
  op: "Mul"
  input: "mul_21"
  input: "add_1034"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1322"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_306"
  op: "Sub"
  input: "ReadVariableOp_1322"
  input: "mul_1650"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_906"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_306"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1323"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_906"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_907"
  op: "AssignVariableOp"
  input: "Mul_1645/ReadVariableOp/Enter"
  input: "add_1031"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1324"
  op: "ReadVariableOp"
  input: "Mul_1645/ReadVariableOp/Enter"
  input: "^AssignVariableOp_907"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_908"
  op: "AssignVariableOp"
  input: "Mul_1647/ReadVariableOp/Enter"
  input: "add_1032"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1325"
  op: "ReadVariableOp"
  input: "Mul_1647/ReadVariableOp/Enter"
  input: "^AssignVariableOp_908"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1651/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1651/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1651/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1651/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1651"
  op: "Mul"
  input: "Mul_1651/x"
  input: "Mul_1651/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1652/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1652"
  op: "Mul"
  input: "Mul_1652/x"
  input: "CrossReplicaSum_303"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1035"
  op: "AddV2"
  input: "Mul_1651"
  input: "Mul_1652"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1653/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1653/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1653/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1653/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1653"
  op: "Mul"
  input: "Mul_1653/x"
  input: "Mul_1653/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_303"
  op: "Square"
  input: "CrossReplicaSum_303"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1654/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1654"
  op: "Mul"
  input: "Mul_1654/x"
  input: "Square_303"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1036"
  op: "AddV2"
  input: "Mul_1653"
  input: "Mul_1654"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_303"
  op: "Sqrt"
  input: "add_1036"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1037/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1037"
  op: "AddV2"
  input: "Sqrt_303"
  input: "add_1037/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_306"
  op: "RealDiv"
  input: "add_1035"
  input: "add_1037"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1655"
  op: "Mul"
  input: "mul_21"
  input: "truediv_306"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1326"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_307"
  op: "Sub"
  input: "ReadVariableOp_1326"
  input: "mul_1655"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_909"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_307"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1327"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_909"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_910"
  op: "AssignVariableOp"
  input: "Mul_1651/ReadVariableOp/Enter"
  input: "add_1035"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1328"
  op: "ReadVariableOp"
  input: "Mul_1651/ReadVariableOp/Enter"
  input: "^AssignVariableOp_910"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_911"
  op: "AssignVariableOp"
  input: "Mul_1653/ReadVariableOp/Enter"
  input: "add_1036"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1329"
  op: "ReadVariableOp"
  input: "Mul_1653/ReadVariableOp/Enter"
  input: "^AssignVariableOp_911"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1656/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1656/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1656/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1656/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1656"
  op: "Mul"
  input: "Mul_1656/x"
  input: "Mul_1656/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1657/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1657"
  op: "Mul"
  input: "Mul_1657/x"
  input: "CrossReplicaSum_304"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1038"
  op: "AddV2"
  input: "Mul_1656"
  input: "Mul_1657"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1658/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1658/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1658/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1658/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1658"
  op: "Mul"
  input: "Mul_1658/x"
  input: "Mul_1658/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_304"
  op: "Square"
  input: "CrossReplicaSum_304"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1659/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1659"
  op: "Mul"
  input: "Mul_1659/x"
  input: "Square_304"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1039"
  op: "AddV2"
  input: "Mul_1658"
  input: "Mul_1659"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_304"
  op: "Sqrt"
  input: "add_1039"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1040/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1040"
  op: "AddV2"
  input: "Sqrt_304"
  input: "add_1040/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_307"
  op: "RealDiv"
  input: "add_1038"
  input: "add_1040"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1330"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1660/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1660"
  op: "Mul"
  input: "mul_1660/x"
  input: "ReadVariableOp_1330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1041"
  op: "AddV2"
  input: "truediv_307"
  input: "mul_1660"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1661"
  op: "Mul"
  input: "mul_21"
  input: "add_1041"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1331"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_308"
  op: "Sub"
  input: "ReadVariableOp_1331"
  input: "mul_1661"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_912"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_308"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1332"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_912"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_913"
  op: "AssignVariableOp"
  input: "Mul_1656/ReadVariableOp/Enter"
  input: "add_1038"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1333"
  op: "ReadVariableOp"
  input: "Mul_1656/ReadVariableOp/Enter"
  input: "^AssignVariableOp_913"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_914"
  op: "AssignVariableOp"
  input: "Mul_1658/ReadVariableOp/Enter"
  input: "add_1039"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1334"
  op: "ReadVariableOp"
  input: "Mul_1658/ReadVariableOp/Enter"
  input: "^AssignVariableOp_914"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1662/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1662/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1662/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1662/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1662"
  op: "Mul"
  input: "Mul_1662/x"
  input: "Mul_1662/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1663/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1663"
  op: "Mul"
  input: "Mul_1663/x"
  input: "CrossReplicaSum_305"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1042"
  op: "AddV2"
  input: "Mul_1662"
  input: "Mul_1663"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1664/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1664/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1664/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1664/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1664"
  op: "Mul"
  input: "Mul_1664/x"
  input: "Mul_1664/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_305"
  op: "Square"
  input: "CrossReplicaSum_305"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1665/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1665"
  op: "Mul"
  input: "Mul_1665/x"
  input: "Square_305"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1043"
  op: "AddV2"
  input: "Mul_1664"
  input: "Mul_1665"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_305"
  op: "Sqrt"
  input: "add_1043"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1044/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1044"
  op: "AddV2"
  input: "Sqrt_305"
  input: "add_1044/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_308"
  op: "RealDiv"
  input: "add_1042"
  input: "add_1044"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1666"
  op: "Mul"
  input: "mul_21"
  input: "truediv_308"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1335"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_309"
  op: "Sub"
  input: "ReadVariableOp_1335"
  input: "mul_1666"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_915"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_309"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1336"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_915"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_916"
  op: "AssignVariableOp"
  input: "Mul_1662/ReadVariableOp/Enter"
  input: "add_1042"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1337"
  op: "ReadVariableOp"
  input: "Mul_1662/ReadVariableOp/Enter"
  input: "^AssignVariableOp_916"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_917"
  op: "AssignVariableOp"
  input: "Mul_1664/ReadVariableOp/Enter"
  input: "add_1043"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1338"
  op: "ReadVariableOp"
  input: "Mul_1664/ReadVariableOp/Enter"
  input: "^AssignVariableOp_917"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1667/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1667/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1667/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1667/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1667"
  op: "Mul"
  input: "Mul_1667/x"
  input: "Mul_1667/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1668/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1668"
  op: "Mul"
  input: "Mul_1668/x"
  input: "CrossReplicaSum_306"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1045"
  op: "AddV2"
  input: "Mul_1667"
  input: "Mul_1668"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1669/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1669/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1669/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1669/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1669"
  op: "Mul"
  input: "Mul_1669/x"
  input: "Mul_1669/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_306"
  op: "Square"
  input: "CrossReplicaSum_306"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1670/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1670"
  op: "Mul"
  input: "Mul_1670/x"
  input: "Square_306"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1046"
  op: "AddV2"
  input: "Mul_1669"
  input: "Mul_1670"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_306"
  op: "Sqrt"
  input: "add_1046"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1047/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1047"
  op: "AddV2"
  input: "Sqrt_306"
  input: "add_1047/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_309"
  op: "RealDiv"
  input: "add_1045"
  input: "add_1047"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1339"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1671/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1671"
  op: "Mul"
  input: "mul_1671/x"
  input: "ReadVariableOp_1339"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1048"
  op: "AddV2"
  input: "truediv_309"
  input: "mul_1671"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1672"
  op: "Mul"
  input: "mul_21"
  input: "add_1048"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1340"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_310"
  op: "Sub"
  input: "ReadVariableOp_1340"
  input: "mul_1672"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_918"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_310"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1341"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_918"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_919"
  op: "AssignVariableOp"
  input: "Mul_1667/ReadVariableOp/Enter"
  input: "add_1045"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1342"
  op: "ReadVariableOp"
  input: "Mul_1667/ReadVariableOp/Enter"
  input: "^AssignVariableOp_919"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_920"
  op: "AssignVariableOp"
  input: "Mul_1669/ReadVariableOp/Enter"
  input: "add_1046"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1343"
  op: "ReadVariableOp"
  input: "Mul_1669/ReadVariableOp/Enter"
  input: "^AssignVariableOp_920"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1673/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1673/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1673/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1673/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1673"
  op: "Mul"
  input: "Mul_1673/x"
  input: "Mul_1673/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1674/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1674"
  op: "Mul"
  input: "Mul_1674/x"
  input: "CrossReplicaSum_307"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1049"
  op: "AddV2"
  input: "Mul_1673"
  input: "Mul_1674"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1675/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1675/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1675/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1675/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1675"
  op: "Mul"
  input: "Mul_1675/x"
  input: "Mul_1675/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_307"
  op: "Square"
  input: "CrossReplicaSum_307"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1676/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1676"
  op: "Mul"
  input: "Mul_1676/x"
  input: "Square_307"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1050"
  op: "AddV2"
  input: "Mul_1675"
  input: "Mul_1676"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_307"
  op: "Sqrt"
  input: "add_1050"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1051/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1051"
  op: "AddV2"
  input: "Sqrt_307"
  input: "add_1051/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_310"
  op: "RealDiv"
  input: "add_1049"
  input: "add_1051"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1677"
  op: "Mul"
  input: "mul_21"
  input: "truediv_310"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1344"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_311"
  op: "Sub"
  input: "ReadVariableOp_1344"
  input: "mul_1677"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_921"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_311"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1345"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_921"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_922"
  op: "AssignVariableOp"
  input: "Mul_1673/ReadVariableOp/Enter"
  input: "add_1049"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1346"
  op: "ReadVariableOp"
  input: "Mul_1673/ReadVariableOp/Enter"
  input: "^AssignVariableOp_922"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_923"
  op: "AssignVariableOp"
  input: "Mul_1675/ReadVariableOp/Enter"
  input: "add_1050"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1347"
  op: "ReadVariableOp"
  input: "Mul_1675/ReadVariableOp/Enter"
  input: "^AssignVariableOp_923"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1678/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1678/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1678/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1678/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1678"
  op: "Mul"
  input: "Mul_1678/x"
  input: "Mul_1678/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1679/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1679"
  op: "Mul"
  input: "Mul_1679/x"
  input: "CrossReplicaSum_308"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1052"
  op: "AddV2"
  input: "Mul_1678"
  input: "Mul_1679"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1680/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1680/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1680/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1680/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1680"
  op: "Mul"
  input: "Mul_1680/x"
  input: "Mul_1680/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_308"
  op: "Square"
  input: "CrossReplicaSum_308"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1681/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1681"
  op: "Mul"
  input: "Mul_1681/x"
  input: "Square_308"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1053"
  op: "AddV2"
  input: "Mul_1680"
  input: "Mul_1681"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_308"
  op: "Sqrt"
  input: "add_1053"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1054/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1054"
  op: "AddV2"
  input: "Sqrt_308"
  input: "add_1054/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_311"
  op: "RealDiv"
  input: "add_1052"
  input: "add_1054"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1348"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1682/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1682"
  op: "Mul"
  input: "mul_1682/x"
  input: "ReadVariableOp_1348"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1055"
  op: "AddV2"
  input: "truediv_311"
  input: "mul_1682"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1683"
  op: "Mul"
  input: "mul_21"
  input: "add_1055"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1349"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_312"
  op: "Sub"
  input: "ReadVariableOp_1349"
  input: "mul_1683"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_924"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_312"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1350"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_924"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_925"
  op: "AssignVariableOp"
  input: "Mul_1678/ReadVariableOp/Enter"
  input: "add_1052"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1351"
  op: "ReadVariableOp"
  input: "Mul_1678/ReadVariableOp/Enter"
  input: "^AssignVariableOp_925"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_926"
  op: "AssignVariableOp"
  input: "Mul_1680/ReadVariableOp/Enter"
  input: "add_1053"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1352"
  op: "ReadVariableOp"
  input: "Mul_1680/ReadVariableOp/Enter"
  input: "^AssignVariableOp_926"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1684/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1684/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1684/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1684/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1684"
  op: "Mul"
  input: "Mul_1684/x"
  input: "Mul_1684/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1685/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1685"
  op: "Mul"
  input: "Mul_1685/x"
  input: "CrossReplicaSum_309"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1056"
  op: "AddV2"
  input: "Mul_1684"
  input: "Mul_1685"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1686/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1686/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1686/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1686/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1686"
  op: "Mul"
  input: "Mul_1686/x"
  input: "Mul_1686/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_309"
  op: "Square"
  input: "CrossReplicaSum_309"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1687/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1687"
  op: "Mul"
  input: "Mul_1687/x"
  input: "Square_309"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1057"
  op: "AddV2"
  input: "Mul_1686"
  input: "Mul_1687"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_309"
  op: "Sqrt"
  input: "add_1057"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1058/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1058"
  op: "AddV2"
  input: "Sqrt_309"
  input: "add_1058/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_312"
  op: "RealDiv"
  input: "add_1056"
  input: "add_1058"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1688"
  op: "Mul"
  input: "mul_21"
  input: "truediv_312"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1353"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_313"
  op: "Sub"
  input: "ReadVariableOp_1353"
  input: "mul_1688"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_927"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_313"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1354"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_927"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_928"
  op: "AssignVariableOp"
  input: "Mul_1684/ReadVariableOp/Enter"
  input: "add_1056"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1355"
  op: "ReadVariableOp"
  input: "Mul_1684/ReadVariableOp/Enter"
  input: "^AssignVariableOp_928"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_929"
  op: "AssignVariableOp"
  input: "Mul_1686/ReadVariableOp/Enter"
  input: "add_1057"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1356"
  op: "ReadVariableOp"
  input: "Mul_1686/ReadVariableOp/Enter"
  input: "^AssignVariableOp_929"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1689/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1689/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1689/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1689/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1689"
  op: "Mul"
  input: "Mul_1689/x"
  input: "Mul_1689/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1690/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1690"
  op: "Mul"
  input: "Mul_1690/x"
  input: "CrossReplicaSum_310"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1059"
  op: "AddV2"
  input: "Mul_1689"
  input: "Mul_1690"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1691/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1691/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1691/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1691/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1691"
  op: "Mul"
  input: "Mul_1691/x"
  input: "Mul_1691/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_310"
  op: "Square"
  input: "CrossReplicaSum_310"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1692/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1692"
  op: "Mul"
  input: "Mul_1692/x"
  input: "Square_310"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1060"
  op: "AddV2"
  input: "Mul_1691"
  input: "Mul_1692"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_310"
  op: "Sqrt"
  input: "add_1060"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1061/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1061"
  op: "AddV2"
  input: "Sqrt_310"
  input: "add_1061/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_313"
  op: "RealDiv"
  input: "add_1059"
  input: "add_1061"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1693"
  op: "Mul"
  input: "mul_21"
  input: "truediv_313"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1357"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_314"
  op: "Sub"
  input: "ReadVariableOp_1357"
  input: "mul_1693"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_930"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_314"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1358"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_930"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_931"
  op: "AssignVariableOp"
  input: "Mul_1689/ReadVariableOp/Enter"
  input: "add_1059"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1359"
  op: "ReadVariableOp"
  input: "Mul_1689/ReadVariableOp/Enter"
  input: "^AssignVariableOp_931"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_932"
  op: "AssignVariableOp"
  input: "Mul_1691/ReadVariableOp/Enter"
  input: "add_1060"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1360"
  op: "ReadVariableOp"
  input: "Mul_1691/ReadVariableOp/Enter"
  input: "^AssignVariableOp_932"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1694/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1694/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1694/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1694/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1694"
  op: "Mul"
  input: "Mul_1694/x"
  input: "Mul_1694/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1695/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1695"
  op: "Mul"
  input: "Mul_1695/x"
  input: "CrossReplicaSum_311"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1062"
  op: "AddV2"
  input: "Mul_1694"
  input: "Mul_1695"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1696/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1696/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1696/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1696/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1696"
  op: "Mul"
  input: "Mul_1696/x"
  input: "Mul_1696/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_311"
  op: "Square"
  input: "CrossReplicaSum_311"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1697/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1697"
  op: "Mul"
  input: "Mul_1697/x"
  input: "Square_311"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1063"
  op: "AddV2"
  input: "Mul_1696"
  input: "Mul_1697"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_311"
  op: "Sqrt"
  input: "add_1063"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1064/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1064"
  op: "AddV2"
  input: "Sqrt_311"
  input: "add_1064/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_314"
  op: "RealDiv"
  input: "add_1062"
  input: "add_1064"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1698"
  op: "Mul"
  input: "mul_21"
  input: "truediv_314"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1361"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_315"
  op: "Sub"
  input: "ReadVariableOp_1361"
  input: "mul_1698"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_933"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_315"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1362"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_933"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_934"
  op: "AssignVariableOp"
  input: "Mul_1694/ReadVariableOp/Enter"
  input: "add_1062"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1363"
  op: "ReadVariableOp"
  input: "Mul_1694/ReadVariableOp/Enter"
  input: "^AssignVariableOp_934"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_935"
  op: "AssignVariableOp"
  input: "Mul_1696/ReadVariableOp/Enter"
  input: "add_1063"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1364"
  op: "ReadVariableOp"
  input: "Mul_1696/ReadVariableOp/Enter"
  input: "^AssignVariableOp_935"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1699/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1699/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1699/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1699/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1699"
  op: "Mul"
  input: "Mul_1699/x"
  input: "Mul_1699/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1700/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1700"
  op: "Mul"
  input: "Mul_1700/x"
  input: "CrossReplicaSum_312"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1065"
  op: "AddV2"
  input: "Mul_1699"
  input: "Mul_1700"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1701/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1701/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1701/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1701/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1701"
  op: "Mul"
  input: "Mul_1701/x"
  input: "Mul_1701/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_312"
  op: "Square"
  input: "CrossReplicaSum_312"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1702/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1702"
  op: "Mul"
  input: "Mul_1702/x"
  input: "Square_312"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1066"
  op: "AddV2"
  input: "Mul_1701"
  input: "Mul_1702"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_312"
  op: "Sqrt"
  input: "add_1066"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1067/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1067"
  op: "AddV2"
  input: "Sqrt_312"
  input: "add_1067/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_315"
  op: "RealDiv"
  input: "add_1065"
  input: "add_1067"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1365"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1703/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1703"
  op: "Mul"
  input: "mul_1703/x"
  input: "ReadVariableOp_1365"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1068"
  op: "AddV2"
  input: "truediv_315"
  input: "mul_1703"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1704"
  op: "Mul"
  input: "mul_21"
  input: "add_1068"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1366"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_316"
  op: "Sub"
  input: "ReadVariableOp_1366"
  input: "mul_1704"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_936"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_316"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1367"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_936"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_937"
  op: "AssignVariableOp"
  input: "Mul_1699/ReadVariableOp/Enter"
  input: "add_1065"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1368"
  op: "ReadVariableOp"
  input: "Mul_1699/ReadVariableOp/Enter"
  input: "^AssignVariableOp_937"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_938"
  op: "AssignVariableOp"
  input: "Mul_1701/ReadVariableOp/Enter"
  input: "add_1066"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1369"
  op: "ReadVariableOp"
  input: "Mul_1701/ReadVariableOp/Enter"
  input: "^AssignVariableOp_938"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1705/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1705/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1705/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1705/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1705"
  op: "Mul"
  input: "Mul_1705/x"
  input: "Mul_1705/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1706/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1706"
  op: "Mul"
  input: "Mul_1706/x"
  input: "CrossReplicaSum_313"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1069"
  op: "AddV2"
  input: "Mul_1705"
  input: "Mul_1706"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1707/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1707/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1707/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1707/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1707"
  op: "Mul"
  input: "Mul_1707/x"
  input: "Mul_1707/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_313"
  op: "Square"
  input: "CrossReplicaSum_313"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1708/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1708"
  op: "Mul"
  input: "Mul_1708/x"
  input: "Square_313"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1070"
  op: "AddV2"
  input: "Mul_1707"
  input: "Mul_1708"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_313"
  op: "Sqrt"
  input: "add_1070"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1071/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1071"
  op: "AddV2"
  input: "Sqrt_313"
  input: "add_1071/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_316"
  op: "RealDiv"
  input: "add_1069"
  input: "add_1071"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1709"
  op: "Mul"
  input: "mul_21"
  input: "truediv_316"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1370"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_317"
  op: "Sub"
  input: "ReadVariableOp_1370"
  input: "mul_1709"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_939"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_317"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1371"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_939"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_940"
  op: "AssignVariableOp"
  input: "Mul_1705/ReadVariableOp/Enter"
  input: "add_1069"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1372"
  op: "ReadVariableOp"
  input: "Mul_1705/ReadVariableOp/Enter"
  input: "^AssignVariableOp_940"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_941"
  op: "AssignVariableOp"
  input: "Mul_1707/ReadVariableOp/Enter"
  input: "add_1070"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1373"
  op: "ReadVariableOp"
  input: "Mul_1707/ReadVariableOp/Enter"
  input: "^AssignVariableOp_941"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1710/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1710/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1710/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1710/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1710"
  op: "Mul"
  input: "Mul_1710/x"
  input: "Mul_1710/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1711/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1711"
  op: "Mul"
  input: "Mul_1711/x"
  input: "CrossReplicaSum_314"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1072"
  op: "AddV2"
  input: "Mul_1710"
  input: "Mul_1711"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1712/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1712/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1712/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1712/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1712"
  op: "Mul"
  input: "Mul_1712/x"
  input: "Mul_1712/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_314"
  op: "Square"
  input: "CrossReplicaSum_314"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1713/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1713"
  op: "Mul"
  input: "Mul_1713/x"
  input: "Square_314"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1073"
  op: "AddV2"
  input: "Mul_1712"
  input: "Mul_1713"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_314"
  op: "Sqrt"
  input: "add_1073"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1074/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1074"
  op: "AddV2"
  input: "Sqrt_314"
  input: "add_1074/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_317"
  op: "RealDiv"
  input: "add_1072"
  input: "add_1074"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1374"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1714/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1714"
  op: "Mul"
  input: "mul_1714/x"
  input: "ReadVariableOp_1374"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1075"
  op: "AddV2"
  input: "truediv_317"
  input: "mul_1714"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1715"
  op: "Mul"
  input: "mul_21"
  input: "add_1075"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1375"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_318"
  op: "Sub"
  input: "ReadVariableOp_1375"
  input: "mul_1715"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_942"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_318"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1376"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_942"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_943"
  op: "AssignVariableOp"
  input: "Mul_1710/ReadVariableOp/Enter"
  input: "add_1072"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1377"
  op: "ReadVariableOp"
  input: "Mul_1710/ReadVariableOp/Enter"
  input: "^AssignVariableOp_943"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_944"
  op: "AssignVariableOp"
  input: "Mul_1712/ReadVariableOp/Enter"
  input: "add_1073"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1378"
  op: "ReadVariableOp"
  input: "Mul_1712/ReadVariableOp/Enter"
  input: "^AssignVariableOp_944"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_m"
  input: "electra/encoder/layer_6/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_v"
  input: "electra/encoder/layer_6/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1716/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1716/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1716/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1716/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1716"
  op: "Mul"
  input: "Mul_1716/x"
  input: "Mul_1716/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1717/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1717"
  op: "Mul"
  input: "Mul_1717/x"
  input: "CrossReplicaSum_315"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1076"
  op: "AddV2"
  input: "Mul_1716"
  input: "Mul_1717"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1718/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1718/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1718/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1718/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1718"
  op: "Mul"
  input: "Mul_1718/x"
  input: "Mul_1718/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_315"
  op: "Square"
  input: "CrossReplicaSum_315"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1719/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1719"
  op: "Mul"
  input: "Mul_1719/x"
  input: "Square_315"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1077"
  op: "AddV2"
  input: "Mul_1718"
  input: "Mul_1719"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_315"
  op: "Sqrt"
  input: "add_1077"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1078/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1078"
  op: "AddV2"
  input: "Sqrt_315"
  input: "add_1078/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_318"
  op: "RealDiv"
  input: "add_1076"
  input: "add_1078"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1720"
  op: "Mul"
  input: "mul_21"
  input: "truediv_318"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1379"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_319"
  op: "Sub"
  input: "ReadVariableOp_1379"
  input: "mul_1720"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_945"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_319"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1380"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_945"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_946"
  op: "AssignVariableOp"
  input: "Mul_1716/ReadVariableOp/Enter"
  input: "add_1076"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1381"
  op: "ReadVariableOp"
  input: "Mul_1716/ReadVariableOp/Enter"
  input: "^AssignVariableOp_946"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_947"
  op: "AssignVariableOp"
  input: "Mul_1718/ReadVariableOp/Enter"
  input: "add_1077"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1382"
  op: "ReadVariableOp"
  input: "Mul_1718/ReadVariableOp/Enter"
  input: "^AssignVariableOp_947"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1721/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1721/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1721/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1721/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1721"
  op: "Mul"
  input: "Mul_1721/x"
  input: "Mul_1721/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1722/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1722"
  op: "Mul"
  input: "Mul_1722/x"
  input: "CrossReplicaSum_316"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1079"
  op: "AddV2"
  input: "Mul_1721"
  input: "Mul_1722"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1723/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1723/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1723/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1723/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1723"
  op: "Mul"
  input: "Mul_1723/x"
  input: "Mul_1723/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_316"
  op: "Square"
  input: "CrossReplicaSum_316"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1724/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1724"
  op: "Mul"
  input: "Mul_1724/x"
  input: "Square_316"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1080"
  op: "AddV2"
  input: "Mul_1723"
  input: "Mul_1724"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_316"
  op: "Sqrt"
  input: "add_1080"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1081/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1081"
  op: "AddV2"
  input: "Sqrt_316"
  input: "add_1081/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_319"
  op: "RealDiv"
  input: "add_1079"
  input: "add_1081"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1725"
  op: "Mul"
  input: "mul_21"
  input: "truediv_319"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1383"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_320"
  op: "Sub"
  input: "ReadVariableOp_1383"
  input: "mul_1725"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_948"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_320"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1384"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_948"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_949"
  op: "AssignVariableOp"
  input: "Mul_1721/ReadVariableOp/Enter"
  input: "add_1079"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1385"
  op: "ReadVariableOp"
  input: "Mul_1721/ReadVariableOp/Enter"
  input: "^AssignVariableOp_949"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_950"
  op: "AssignVariableOp"
  input: "Mul_1723/ReadVariableOp/Enter"
  input: "add_1080"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1386"
  op: "ReadVariableOp"
  input: "Mul_1723/ReadVariableOp/Enter"
  input: "^AssignVariableOp_950"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1726/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1726/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1726/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1726/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1726"
  op: "Mul"
  input: "Mul_1726/x"
  input: "Mul_1726/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1727/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1727"
  op: "Mul"
  input: "Mul_1727/x"
  input: "CrossReplicaSum_317"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1082"
  op: "AddV2"
  input: "Mul_1726"
  input: "Mul_1727"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1728/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1728/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1728/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1728/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1728"
  op: "Mul"
  input: "Mul_1728/x"
  input: "Mul_1728/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_317"
  op: "Square"
  input: "CrossReplicaSum_317"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1729/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1729"
  op: "Mul"
  input: "Mul_1729/x"
  input: "Square_317"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1083"
  op: "AddV2"
  input: "Mul_1728"
  input: "Mul_1729"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_317"
  op: "Sqrt"
  input: "add_1083"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1084/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1084"
  op: "AddV2"
  input: "Sqrt_317"
  input: "add_1084/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_320"
  op: "RealDiv"
  input: "add_1082"
  input: "add_1084"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1730"
  op: "Mul"
  input: "mul_21"
  input: "truediv_320"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1387"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_321"
  op: "Sub"
  input: "ReadVariableOp_1387"
  input: "mul_1730"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_951"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_321"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1388"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_951"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_952"
  op: "AssignVariableOp"
  input: "Mul_1726/ReadVariableOp/Enter"
  input: "add_1082"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1389"
  op: "ReadVariableOp"
  input: "Mul_1726/ReadVariableOp/Enter"
  input: "^AssignVariableOp_952"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_953"
  op: "AssignVariableOp"
  input: "Mul_1728/ReadVariableOp/Enter"
  input: "add_1083"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1390"
  op: "ReadVariableOp"
  input: "Mul_1728/ReadVariableOp/Enter"
  input: "^AssignVariableOp_953"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1731/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1731/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1731/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1731/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1731"
  op: "Mul"
  input: "Mul_1731/x"
  input: "Mul_1731/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1732/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1732"
  op: "Mul"
  input: "Mul_1732/x"
  input: "CrossReplicaSum_318"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1085"
  op: "AddV2"
  input: "Mul_1731"
  input: "Mul_1732"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1733/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1733/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1733/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1733/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1733"
  op: "Mul"
  input: "Mul_1733/x"
  input: "Mul_1733/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_318"
  op: "Square"
  input: "CrossReplicaSum_318"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1734/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1734"
  op: "Mul"
  input: "Mul_1734/x"
  input: "Square_318"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1086"
  op: "AddV2"
  input: "Mul_1733"
  input: "Mul_1734"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_318"
  op: "Sqrt"
  input: "add_1086"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1087/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1087"
  op: "AddV2"
  input: "Sqrt_318"
  input: "add_1087/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_321"
  op: "RealDiv"
  input: "add_1085"
  input: "add_1087"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1391"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1735/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1735"
  op: "Mul"
  input: "mul_1735/x"
  input: "ReadVariableOp_1391"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1088"
  op: "AddV2"
  input: "truediv_321"
  input: "mul_1735"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1736"
  op: "Mul"
  input: "mul_21"
  input: "add_1088"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1392"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_322"
  op: "Sub"
  input: "ReadVariableOp_1392"
  input: "mul_1736"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_954"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_322"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1393"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_954"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_955"
  op: "AssignVariableOp"
  input: "Mul_1731/ReadVariableOp/Enter"
  input: "add_1085"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1394"
  op: "ReadVariableOp"
  input: "Mul_1731/ReadVariableOp/Enter"
  input: "^AssignVariableOp_955"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_956"
  op: "AssignVariableOp"
  input: "Mul_1733/ReadVariableOp/Enter"
  input: "add_1086"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1395"
  op: "ReadVariableOp"
  input: "Mul_1733/ReadVariableOp/Enter"
  input: "^AssignVariableOp_956"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1737/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1737/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1737/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1737/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1737"
  op: "Mul"
  input: "Mul_1737/x"
  input: "Mul_1737/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1738/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1738"
  op: "Mul"
  input: "Mul_1738/x"
  input: "CrossReplicaSum_319"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1089"
  op: "AddV2"
  input: "Mul_1737"
  input: "Mul_1738"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1739/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1739/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1739/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1739/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1739"
  op: "Mul"
  input: "Mul_1739/x"
  input: "Mul_1739/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_319"
  op: "Square"
  input: "CrossReplicaSum_319"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1740/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1740"
  op: "Mul"
  input: "Mul_1740/x"
  input: "Square_319"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1090"
  op: "AddV2"
  input: "Mul_1739"
  input: "Mul_1740"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_319"
  op: "Sqrt"
  input: "add_1090"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1091/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1091"
  op: "AddV2"
  input: "Sqrt_319"
  input: "add_1091/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_322"
  op: "RealDiv"
  input: "add_1089"
  input: "add_1091"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1741"
  op: "Mul"
  input: "mul_21"
  input: "truediv_322"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1396"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_323"
  op: "Sub"
  input: "ReadVariableOp_1396"
  input: "mul_1741"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_957"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_323"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1397"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_957"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_958"
  op: "AssignVariableOp"
  input: "Mul_1737/ReadVariableOp/Enter"
  input: "add_1089"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1398"
  op: "ReadVariableOp"
  input: "Mul_1737/ReadVariableOp/Enter"
  input: "^AssignVariableOp_958"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_959"
  op: "AssignVariableOp"
  input: "Mul_1739/ReadVariableOp/Enter"
  input: "add_1090"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1399"
  op: "ReadVariableOp"
  input: "Mul_1739/ReadVariableOp/Enter"
  input: "^AssignVariableOp_959"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1742/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1742/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1742/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1742/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1742"
  op: "Mul"
  input: "Mul_1742/x"
  input: "Mul_1742/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1743/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1743"
  op: "Mul"
  input: "Mul_1743/x"
  input: "CrossReplicaSum_320"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1092"
  op: "AddV2"
  input: "Mul_1742"
  input: "Mul_1743"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1744/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1744/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1744/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1744/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1744"
  op: "Mul"
  input: "Mul_1744/x"
  input: "Mul_1744/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_320"
  op: "Square"
  input: "CrossReplicaSum_320"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1745/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1745"
  op: "Mul"
  input: "Mul_1745/x"
  input: "Square_320"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1093"
  op: "AddV2"
  input: "Mul_1744"
  input: "Mul_1745"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_320"
  op: "Sqrt"
  input: "add_1093"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1094/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1094"
  op: "AddV2"
  input: "Sqrt_320"
  input: "add_1094/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_323"
  op: "RealDiv"
  input: "add_1092"
  input: "add_1094"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1400"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1746/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1746"
  op: "Mul"
  input: "mul_1746/x"
  input: "ReadVariableOp_1400"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1095"
  op: "AddV2"
  input: "truediv_323"
  input: "mul_1746"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1747"
  op: "Mul"
  input: "mul_21"
  input: "add_1095"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1401"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_324"
  op: "Sub"
  input: "ReadVariableOp_1401"
  input: "mul_1747"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_960"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_324"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1402"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_960"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_961"
  op: "AssignVariableOp"
  input: "Mul_1742/ReadVariableOp/Enter"
  input: "add_1092"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1403"
  op: "ReadVariableOp"
  input: "Mul_1742/ReadVariableOp/Enter"
  input: "^AssignVariableOp_961"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_962"
  op: "AssignVariableOp"
  input: "Mul_1744/ReadVariableOp/Enter"
  input: "add_1093"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1404"
  op: "ReadVariableOp"
  input: "Mul_1744/ReadVariableOp/Enter"
  input: "^AssignVariableOp_962"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1748/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1748/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1748/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1748/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1748"
  op: "Mul"
  input: "Mul_1748/x"
  input: "Mul_1748/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1749/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1749"
  op: "Mul"
  input: "Mul_1749/x"
  input: "CrossReplicaSum_321"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1096"
  op: "AddV2"
  input: "Mul_1748"
  input: "Mul_1749"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1750/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1750/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1750/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1750/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1750"
  op: "Mul"
  input: "Mul_1750/x"
  input: "Mul_1750/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_321"
  op: "Square"
  input: "CrossReplicaSum_321"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1751/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1751"
  op: "Mul"
  input: "Mul_1751/x"
  input: "Square_321"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1097"
  op: "AddV2"
  input: "Mul_1750"
  input: "Mul_1751"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_321"
  op: "Sqrt"
  input: "add_1097"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1098/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1098"
  op: "AddV2"
  input: "Sqrt_321"
  input: "add_1098/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_324"
  op: "RealDiv"
  input: "add_1096"
  input: "add_1098"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1752"
  op: "Mul"
  input: "mul_21"
  input: "truediv_324"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1405"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_325"
  op: "Sub"
  input: "ReadVariableOp_1405"
  input: "mul_1752"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_963"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_325"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1406"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_963"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_964"
  op: "AssignVariableOp"
  input: "Mul_1748/ReadVariableOp/Enter"
  input: "add_1096"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1407"
  op: "ReadVariableOp"
  input: "Mul_1748/ReadVariableOp/Enter"
  input: "^AssignVariableOp_964"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_965"
  op: "AssignVariableOp"
  input: "Mul_1750/ReadVariableOp/Enter"
  input: "add_1097"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1408"
  op: "ReadVariableOp"
  input: "Mul_1750/ReadVariableOp/Enter"
  input: "^AssignVariableOp_965"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1753/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1753/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1753/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1753/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1753"
  op: "Mul"
  input: "Mul_1753/x"
  input: "Mul_1753/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1754/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1754"
  op: "Mul"
  input: "Mul_1754/x"
  input: "CrossReplicaSum_322"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1099"
  op: "AddV2"
  input: "Mul_1753"
  input: "Mul_1754"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1755/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1755/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1755/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1755/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1755"
  op: "Mul"
  input: "Mul_1755/x"
  input: "Mul_1755/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_322"
  op: "Square"
  input: "CrossReplicaSum_322"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1756/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1756"
  op: "Mul"
  input: "Mul_1756/x"
  input: "Square_322"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1100"
  op: "AddV2"
  input: "Mul_1755"
  input: "Mul_1756"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_322"
  op: "Sqrt"
  input: "add_1100"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1101/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1101"
  op: "AddV2"
  input: "Sqrt_322"
  input: "add_1101/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_325"
  op: "RealDiv"
  input: "add_1099"
  input: "add_1101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1409"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1757/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1757"
  op: "Mul"
  input: "mul_1757/x"
  input: "ReadVariableOp_1409"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1102"
  op: "AddV2"
  input: "truediv_325"
  input: "mul_1757"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1758"
  op: "Mul"
  input: "mul_21"
  input: "add_1102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1410"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_326"
  op: "Sub"
  input: "ReadVariableOp_1410"
  input: "mul_1758"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_966"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_326"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1411"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_966"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_967"
  op: "AssignVariableOp"
  input: "Mul_1753/ReadVariableOp/Enter"
  input: "add_1099"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1412"
  op: "ReadVariableOp"
  input: "Mul_1753/ReadVariableOp/Enter"
  input: "^AssignVariableOp_967"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_968"
  op: "AssignVariableOp"
  input: "Mul_1755/ReadVariableOp/Enter"
  input: "add_1100"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1413"
  op: "ReadVariableOp"
  input: "Mul_1755/ReadVariableOp/Enter"
  input: "^AssignVariableOp_968"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1759/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1759/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1759/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1759/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1759"
  op: "Mul"
  input: "Mul_1759/x"
  input: "Mul_1759/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1760/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1760"
  op: "Mul"
  input: "Mul_1760/x"
  input: "CrossReplicaSum_323"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1103"
  op: "AddV2"
  input: "Mul_1759"
  input: "Mul_1760"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1761/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1761/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1761/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1761/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1761"
  op: "Mul"
  input: "Mul_1761/x"
  input: "Mul_1761/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_323"
  op: "Square"
  input: "CrossReplicaSum_323"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1762/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1762"
  op: "Mul"
  input: "Mul_1762/x"
  input: "Square_323"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1104"
  op: "AddV2"
  input: "Mul_1761"
  input: "Mul_1762"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_323"
  op: "Sqrt"
  input: "add_1104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1105/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1105"
  op: "AddV2"
  input: "Sqrt_323"
  input: "add_1105/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_326"
  op: "RealDiv"
  input: "add_1103"
  input: "add_1105"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1763"
  op: "Mul"
  input: "mul_21"
  input: "truediv_326"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1414"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_327"
  op: "Sub"
  input: "ReadVariableOp_1414"
  input: "mul_1763"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_969"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_327"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1415"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_969"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_970"
  op: "AssignVariableOp"
  input: "Mul_1759/ReadVariableOp/Enter"
  input: "add_1103"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1416"
  op: "ReadVariableOp"
  input: "Mul_1759/ReadVariableOp/Enter"
  input: "^AssignVariableOp_970"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_971"
  op: "AssignVariableOp"
  input: "Mul_1761/ReadVariableOp/Enter"
  input: "add_1104"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1417"
  op: "ReadVariableOp"
  input: "Mul_1761/ReadVariableOp/Enter"
  input: "^AssignVariableOp_971"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1764/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1764/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1764/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1764/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1764"
  op: "Mul"
  input: "Mul_1764/x"
  input: "Mul_1764/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1765/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1765"
  op: "Mul"
  input: "Mul_1765/x"
  input: "CrossReplicaSum_324"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1106"
  op: "AddV2"
  input: "Mul_1764"
  input: "Mul_1765"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1766/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1766/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1766/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1766/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1766"
  op: "Mul"
  input: "Mul_1766/x"
  input: "Mul_1766/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_324"
  op: "Square"
  input: "CrossReplicaSum_324"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1767/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1767"
  op: "Mul"
  input: "Mul_1767/x"
  input: "Square_324"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1107"
  op: "AddV2"
  input: "Mul_1766"
  input: "Mul_1767"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_324"
  op: "Sqrt"
  input: "add_1107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1108/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1108"
  op: "AddV2"
  input: "Sqrt_324"
  input: "add_1108/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_327"
  op: "RealDiv"
  input: "add_1106"
  input: "add_1108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1418"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1768/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1768"
  op: "Mul"
  input: "mul_1768/x"
  input: "ReadVariableOp_1418"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1109"
  op: "AddV2"
  input: "truediv_327"
  input: "mul_1768"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1769"
  op: "Mul"
  input: "mul_21"
  input: "add_1109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1419"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_328"
  op: "Sub"
  input: "ReadVariableOp_1419"
  input: "mul_1769"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_972"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_328"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1420"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_972"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_973"
  op: "AssignVariableOp"
  input: "Mul_1764/ReadVariableOp/Enter"
  input: "add_1106"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1421"
  op: "ReadVariableOp"
  input: "Mul_1764/ReadVariableOp/Enter"
  input: "^AssignVariableOp_973"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_974"
  op: "AssignVariableOp"
  input: "Mul_1766/ReadVariableOp/Enter"
  input: "add_1107"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1422"
  op: "ReadVariableOp"
  input: "Mul_1766/ReadVariableOp/Enter"
  input: "^AssignVariableOp_974"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1770/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1770/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1770/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1770/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1770"
  op: "Mul"
  input: "Mul_1770/x"
  input: "Mul_1770/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1771/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1771"
  op: "Mul"
  input: "Mul_1771/x"
  input: "CrossReplicaSum_325"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1110"
  op: "AddV2"
  input: "Mul_1770"
  input: "Mul_1771"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1772/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1772/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1772/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1772/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1772"
  op: "Mul"
  input: "Mul_1772/x"
  input: "Mul_1772/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_325"
  op: "Square"
  input: "CrossReplicaSum_325"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1773/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1773"
  op: "Mul"
  input: "Mul_1773/x"
  input: "Square_325"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1111"
  op: "AddV2"
  input: "Mul_1772"
  input: "Mul_1773"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_325"
  op: "Sqrt"
  input: "add_1111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1112/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1112"
  op: "AddV2"
  input: "Sqrt_325"
  input: "add_1112/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_328"
  op: "RealDiv"
  input: "add_1110"
  input: "add_1112"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1774"
  op: "Mul"
  input: "mul_21"
  input: "truediv_328"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1423"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_329"
  op: "Sub"
  input: "ReadVariableOp_1423"
  input: "mul_1774"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_975"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_329"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1424"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_975"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_976"
  op: "AssignVariableOp"
  input: "Mul_1770/ReadVariableOp/Enter"
  input: "add_1110"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1425"
  op: "ReadVariableOp"
  input: "Mul_1770/ReadVariableOp/Enter"
  input: "^AssignVariableOp_976"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_977"
  op: "AssignVariableOp"
  input: "Mul_1772/ReadVariableOp/Enter"
  input: "add_1111"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1426"
  op: "ReadVariableOp"
  input: "Mul_1772/ReadVariableOp/Enter"
  input: "^AssignVariableOp_977"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1775/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1775/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1775/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1775/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1775"
  op: "Mul"
  input: "Mul_1775/x"
  input: "Mul_1775/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1776/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1776"
  op: "Mul"
  input: "Mul_1776/x"
  input: "CrossReplicaSum_326"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1113"
  op: "AddV2"
  input: "Mul_1775"
  input: "Mul_1776"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1777/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1777/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1777/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1777/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1777"
  op: "Mul"
  input: "Mul_1777/x"
  input: "Mul_1777/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_326"
  op: "Square"
  input: "CrossReplicaSum_326"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1778/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1778"
  op: "Mul"
  input: "Mul_1778/x"
  input: "Square_326"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1114"
  op: "AddV2"
  input: "Mul_1777"
  input: "Mul_1778"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_326"
  op: "Sqrt"
  input: "add_1114"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1115/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1115"
  op: "AddV2"
  input: "Sqrt_326"
  input: "add_1115/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_329"
  op: "RealDiv"
  input: "add_1113"
  input: "add_1115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1779"
  op: "Mul"
  input: "mul_21"
  input: "truediv_329"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1427"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_330"
  op: "Sub"
  input: "ReadVariableOp_1427"
  input: "mul_1779"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_978"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_330"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1428"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_978"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_979"
  op: "AssignVariableOp"
  input: "Mul_1775/ReadVariableOp/Enter"
  input: "add_1113"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1429"
  op: "ReadVariableOp"
  input: "Mul_1775/ReadVariableOp/Enter"
  input: "^AssignVariableOp_979"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_980"
  op: "AssignVariableOp"
  input: "Mul_1777/ReadVariableOp/Enter"
  input: "add_1114"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1430"
  op: "ReadVariableOp"
  input: "Mul_1777/ReadVariableOp/Enter"
  input: "^AssignVariableOp_980"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1780/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1780/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1780/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1780/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1780"
  op: "Mul"
  input: "Mul_1780/x"
  input: "Mul_1780/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1781/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1781"
  op: "Mul"
  input: "Mul_1781/x"
  input: "CrossReplicaSum_327"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1116"
  op: "AddV2"
  input: "Mul_1780"
  input: "Mul_1781"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1782/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1782/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1782/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1782/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1782"
  op: "Mul"
  input: "Mul_1782/x"
  input: "Mul_1782/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_327"
  op: "Square"
  input: "CrossReplicaSum_327"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1783/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1783"
  op: "Mul"
  input: "Mul_1783/x"
  input: "Square_327"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1117"
  op: "AddV2"
  input: "Mul_1782"
  input: "Mul_1783"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_327"
  op: "Sqrt"
  input: "add_1117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1118/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1118"
  op: "AddV2"
  input: "Sqrt_327"
  input: "add_1118/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_330"
  op: "RealDiv"
  input: "add_1116"
  input: "add_1118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1784"
  op: "Mul"
  input: "mul_21"
  input: "truediv_330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1431"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_331"
  op: "Sub"
  input: "ReadVariableOp_1431"
  input: "mul_1784"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_981"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_331"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1432"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_981"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_982"
  op: "AssignVariableOp"
  input: "Mul_1780/ReadVariableOp/Enter"
  input: "add_1116"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1433"
  op: "ReadVariableOp"
  input: "Mul_1780/ReadVariableOp/Enter"
  input: "^AssignVariableOp_982"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_983"
  op: "AssignVariableOp"
  input: "Mul_1782/ReadVariableOp/Enter"
  input: "add_1117"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1434"
  op: "ReadVariableOp"
  input: "Mul_1782/ReadVariableOp/Enter"
  input: "^AssignVariableOp_983"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1785/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1785/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1785/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1785/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1785"
  op: "Mul"
  input: "Mul_1785/x"
  input: "Mul_1785/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1786/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1786"
  op: "Mul"
  input: "Mul_1786/x"
  input: "CrossReplicaSum_328"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1119"
  op: "AddV2"
  input: "Mul_1785"
  input: "Mul_1786"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1787/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1787/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1787/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1787/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1787"
  op: "Mul"
  input: "Mul_1787/x"
  input: "Mul_1787/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_328"
  op: "Square"
  input: "CrossReplicaSum_328"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1788/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1788"
  op: "Mul"
  input: "Mul_1788/x"
  input: "Square_328"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1120"
  op: "AddV2"
  input: "Mul_1787"
  input: "Mul_1788"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_328"
  op: "Sqrt"
  input: "add_1120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1121/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1121"
  op: "AddV2"
  input: "Sqrt_328"
  input: "add_1121/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_331"
  op: "RealDiv"
  input: "add_1119"
  input: "add_1121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1435"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1789/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1789"
  op: "Mul"
  input: "mul_1789/x"
  input: "ReadVariableOp_1435"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1122"
  op: "AddV2"
  input: "truediv_331"
  input: "mul_1789"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1790"
  op: "Mul"
  input: "mul_21"
  input: "add_1122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1436"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_332"
  op: "Sub"
  input: "ReadVariableOp_1436"
  input: "mul_1790"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_984"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_332"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1437"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_984"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_985"
  op: "AssignVariableOp"
  input: "Mul_1785/ReadVariableOp/Enter"
  input: "add_1119"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1438"
  op: "ReadVariableOp"
  input: "Mul_1785/ReadVariableOp/Enter"
  input: "^AssignVariableOp_985"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_986"
  op: "AssignVariableOp"
  input: "Mul_1787/ReadVariableOp/Enter"
  input: "add_1120"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1439"
  op: "ReadVariableOp"
  input: "Mul_1787/ReadVariableOp/Enter"
  input: "^AssignVariableOp_986"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1791/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1791/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1791/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1791/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1791"
  op: "Mul"
  input: "Mul_1791/x"
  input: "Mul_1791/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1792/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1792"
  op: "Mul"
  input: "Mul_1792/x"
  input: "CrossReplicaSum_329"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1123"
  op: "AddV2"
  input: "Mul_1791"
  input: "Mul_1792"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1793/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1793/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1793/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1793/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1793"
  op: "Mul"
  input: "Mul_1793/x"
  input: "Mul_1793/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_329"
  op: "Square"
  input: "CrossReplicaSum_329"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1794/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1794"
  op: "Mul"
  input: "Mul_1794/x"
  input: "Square_329"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1124"
  op: "AddV2"
  input: "Mul_1793"
  input: "Mul_1794"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_329"
  op: "Sqrt"
  input: "add_1124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1125/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1125"
  op: "AddV2"
  input: "Sqrt_329"
  input: "add_1125/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_332"
  op: "RealDiv"
  input: "add_1123"
  input: "add_1125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1795"
  op: "Mul"
  input: "mul_21"
  input: "truediv_332"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1440"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_333"
  op: "Sub"
  input: "ReadVariableOp_1440"
  input: "mul_1795"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_987"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_333"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1441"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_987"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_988"
  op: "AssignVariableOp"
  input: "Mul_1791/ReadVariableOp/Enter"
  input: "add_1123"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1442"
  op: "ReadVariableOp"
  input: "Mul_1791/ReadVariableOp/Enter"
  input: "^AssignVariableOp_988"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_989"
  op: "AssignVariableOp"
  input: "Mul_1793/ReadVariableOp/Enter"
  input: "add_1124"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1443"
  op: "ReadVariableOp"
  input: "Mul_1793/ReadVariableOp/Enter"
  input: "^AssignVariableOp_989"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1796/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1796/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1796/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1796/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1796"
  op: "Mul"
  input: "Mul_1796/x"
  input: "Mul_1796/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1797/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1797"
  op: "Mul"
  input: "Mul_1797/x"
  input: "CrossReplicaSum_330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1126"
  op: "AddV2"
  input: "Mul_1796"
  input: "Mul_1797"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1798/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1798/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1798/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1798/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1798"
  op: "Mul"
  input: "Mul_1798/x"
  input: "Mul_1798/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_330"
  op: "Square"
  input: "CrossReplicaSum_330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1799/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1799"
  op: "Mul"
  input: "Mul_1799/x"
  input: "Square_330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1127"
  op: "AddV2"
  input: "Mul_1798"
  input: "Mul_1799"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_330"
  op: "Sqrt"
  input: "add_1127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1128/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1128"
  op: "AddV2"
  input: "Sqrt_330"
  input: "add_1128/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_333"
  op: "RealDiv"
  input: "add_1126"
  input: "add_1128"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1444"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1800/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1800"
  op: "Mul"
  input: "mul_1800/x"
  input: "ReadVariableOp_1444"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1129"
  op: "AddV2"
  input: "truediv_333"
  input: "mul_1800"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1801"
  op: "Mul"
  input: "mul_21"
  input: "add_1129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1445"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_334"
  op: "Sub"
  input: "ReadVariableOp_1445"
  input: "mul_1801"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_990"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_334"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1446"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_990"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_991"
  op: "AssignVariableOp"
  input: "Mul_1796/ReadVariableOp/Enter"
  input: "add_1126"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1447"
  op: "ReadVariableOp"
  input: "Mul_1796/ReadVariableOp/Enter"
  input: "^AssignVariableOp_991"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_992"
  op: "AssignVariableOp"
  input: "Mul_1798/ReadVariableOp/Enter"
  input: "add_1127"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1448"
  op: "ReadVariableOp"
  input: "Mul_1798/ReadVariableOp/Enter"
  input: "^AssignVariableOp_992"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_m"
  input: "electra/encoder/layer_7/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_v"
  input: "electra/encoder/layer_7/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1802/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1802/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1802/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1802/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1802"
  op: "Mul"
  input: "Mul_1802/x"
  input: "Mul_1802/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1803/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1803"
  op: "Mul"
  input: "Mul_1803/x"
  input: "CrossReplicaSum_331"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1130"
  op: "AddV2"
  input: "Mul_1802"
  input: "Mul_1803"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1804/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1804/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1804/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1804/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1804"
  op: "Mul"
  input: "Mul_1804/x"
  input: "Mul_1804/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_331"
  op: "Square"
  input: "CrossReplicaSum_331"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1805/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1805"
  op: "Mul"
  input: "Mul_1805/x"
  input: "Square_331"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1131"
  op: "AddV2"
  input: "Mul_1804"
  input: "Mul_1805"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_331"
  op: "Sqrt"
  input: "add_1131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1132/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1132"
  op: "AddV2"
  input: "Sqrt_331"
  input: "add_1132/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_334"
  op: "RealDiv"
  input: "add_1130"
  input: "add_1132"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1806"
  op: "Mul"
  input: "mul_21"
  input: "truediv_334"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1449"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_335"
  op: "Sub"
  input: "ReadVariableOp_1449"
  input: "mul_1806"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_993"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_335"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1450"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_993"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_994"
  op: "AssignVariableOp"
  input: "Mul_1802/ReadVariableOp/Enter"
  input: "add_1130"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1451"
  op: "ReadVariableOp"
  input: "Mul_1802/ReadVariableOp/Enter"
  input: "^AssignVariableOp_994"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_995"
  op: "AssignVariableOp"
  input: "Mul_1804/ReadVariableOp/Enter"
  input: "add_1131"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1452"
  op: "ReadVariableOp"
  input: "Mul_1804/ReadVariableOp/Enter"
  input: "^AssignVariableOp_995"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1807/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1807/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1807/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1807/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1807"
  op: "Mul"
  input: "Mul_1807/x"
  input: "Mul_1807/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1808/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1808"
  op: "Mul"
  input: "Mul_1808/x"
  input: "CrossReplicaSum_332"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1133"
  op: "AddV2"
  input: "Mul_1807"
  input: "Mul_1808"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1809/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1809/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1809/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1809/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1809"
  op: "Mul"
  input: "Mul_1809/x"
  input: "Mul_1809/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_332"
  op: "Square"
  input: "CrossReplicaSum_332"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1810/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1810"
  op: "Mul"
  input: "Mul_1810/x"
  input: "Square_332"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1134"
  op: "AddV2"
  input: "Mul_1809"
  input: "Mul_1810"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_332"
  op: "Sqrt"
  input: "add_1134"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1135/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1135"
  op: "AddV2"
  input: "Sqrt_332"
  input: "add_1135/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_335"
  op: "RealDiv"
  input: "add_1133"
  input: "add_1135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1811"
  op: "Mul"
  input: "mul_21"
  input: "truediv_335"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1453"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_336"
  op: "Sub"
  input: "ReadVariableOp_1453"
  input: "mul_1811"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_996"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_336"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1454"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_996"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_997"
  op: "AssignVariableOp"
  input: "Mul_1807/ReadVariableOp/Enter"
  input: "add_1133"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1455"
  op: "ReadVariableOp"
  input: "Mul_1807/ReadVariableOp/Enter"
  input: "^AssignVariableOp_997"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_998"
  op: "AssignVariableOp"
  input: "Mul_1809/ReadVariableOp/Enter"
  input: "add_1134"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1456"
  op: "ReadVariableOp"
  input: "Mul_1809/ReadVariableOp/Enter"
  input: "^AssignVariableOp_998"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1812/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1812/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1812/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1812/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1812"
  op: "Mul"
  input: "Mul_1812/x"
  input: "Mul_1812/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1813/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1813"
  op: "Mul"
  input: "Mul_1813/x"
  input: "CrossReplicaSum_333"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1136"
  op: "AddV2"
  input: "Mul_1812"
  input: "Mul_1813"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1814/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1814/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1814/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1814/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1814"
  op: "Mul"
  input: "Mul_1814/x"
  input: "Mul_1814/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_333"
  op: "Square"
  input: "CrossReplicaSum_333"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1815/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1815"
  op: "Mul"
  input: "Mul_1815/x"
  input: "Square_333"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1137"
  op: "AddV2"
  input: "Mul_1814"
  input: "Mul_1815"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_333"
  op: "Sqrt"
  input: "add_1137"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1138/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1138"
  op: "AddV2"
  input: "Sqrt_333"
  input: "add_1138/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_336"
  op: "RealDiv"
  input: "add_1136"
  input: "add_1138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1816"
  op: "Mul"
  input: "mul_21"
  input: "truediv_336"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1457"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_337"
  op: "Sub"
  input: "ReadVariableOp_1457"
  input: "mul_1816"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_999"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_337"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1458"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_999"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1000"
  op: "AssignVariableOp"
  input: "Mul_1812/ReadVariableOp/Enter"
  input: "add_1136"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1459"
  op: "ReadVariableOp"
  input: "Mul_1812/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1000"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1001"
  op: "AssignVariableOp"
  input: "Mul_1814/ReadVariableOp/Enter"
  input: "add_1137"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1460"
  op: "ReadVariableOp"
  input: "Mul_1814/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1001"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1817/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1817/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1817/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1817/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1817"
  op: "Mul"
  input: "Mul_1817/x"
  input: "Mul_1817/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1818/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1818"
  op: "Mul"
  input: "Mul_1818/x"
  input: "CrossReplicaSum_334"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1139"
  op: "AddV2"
  input: "Mul_1817"
  input: "Mul_1818"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1819/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1819/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1819/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1819/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1819"
  op: "Mul"
  input: "Mul_1819/x"
  input: "Mul_1819/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_334"
  op: "Square"
  input: "CrossReplicaSum_334"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1820/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1820"
  op: "Mul"
  input: "Mul_1820/x"
  input: "Square_334"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1140"
  op: "AddV2"
  input: "Mul_1819"
  input: "Mul_1820"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_334"
  op: "Sqrt"
  input: "add_1140"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1141/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1141"
  op: "AddV2"
  input: "Sqrt_334"
  input: "add_1141/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_337"
  op: "RealDiv"
  input: "add_1139"
  input: "add_1141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1461"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1821/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1821"
  op: "Mul"
  input: "mul_1821/x"
  input: "ReadVariableOp_1461"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1142"
  op: "AddV2"
  input: "truediv_337"
  input: "mul_1821"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1822"
  op: "Mul"
  input: "mul_21"
  input: "add_1142"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1462"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_338"
  op: "Sub"
  input: "ReadVariableOp_1462"
  input: "mul_1822"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1002"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_338"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1463"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1002"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1003"
  op: "AssignVariableOp"
  input: "Mul_1817/ReadVariableOp/Enter"
  input: "add_1139"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1464"
  op: "ReadVariableOp"
  input: "Mul_1817/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1003"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1004"
  op: "AssignVariableOp"
  input: "Mul_1819/ReadVariableOp/Enter"
  input: "add_1140"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1465"
  op: "ReadVariableOp"
  input: "Mul_1819/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1004"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1823/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1823/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1823/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1823/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1823"
  op: "Mul"
  input: "Mul_1823/x"
  input: "Mul_1823/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1824/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1824"
  op: "Mul"
  input: "Mul_1824/x"
  input: "CrossReplicaSum_335"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1143"
  op: "AddV2"
  input: "Mul_1823"
  input: "Mul_1824"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1825/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1825/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1825/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1825/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1825"
  op: "Mul"
  input: "Mul_1825/x"
  input: "Mul_1825/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_335"
  op: "Square"
  input: "CrossReplicaSum_335"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1826/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1826"
  op: "Mul"
  input: "Mul_1826/x"
  input: "Square_335"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1144"
  op: "AddV2"
  input: "Mul_1825"
  input: "Mul_1826"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_335"
  op: "Sqrt"
  input: "add_1144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1145/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1145"
  op: "AddV2"
  input: "Sqrt_335"
  input: "add_1145/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_338"
  op: "RealDiv"
  input: "add_1143"
  input: "add_1145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1827"
  op: "Mul"
  input: "mul_21"
  input: "truediv_338"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1466"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_339"
  op: "Sub"
  input: "ReadVariableOp_1466"
  input: "mul_1827"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1005"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_339"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1467"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1005"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1006"
  op: "AssignVariableOp"
  input: "Mul_1823/ReadVariableOp/Enter"
  input: "add_1143"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1468"
  op: "ReadVariableOp"
  input: "Mul_1823/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1006"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1007"
  op: "AssignVariableOp"
  input: "Mul_1825/ReadVariableOp/Enter"
  input: "add_1144"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1469"
  op: "ReadVariableOp"
  input: "Mul_1825/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1007"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1828/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1828/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1828/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1828/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1828"
  op: "Mul"
  input: "Mul_1828/x"
  input: "Mul_1828/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1829/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1829"
  op: "Mul"
  input: "Mul_1829/x"
  input: "CrossReplicaSum_336"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1146"
  op: "AddV2"
  input: "Mul_1828"
  input: "Mul_1829"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1830/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1830/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1830/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1830/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1830"
  op: "Mul"
  input: "Mul_1830/x"
  input: "Mul_1830/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_336"
  op: "Square"
  input: "CrossReplicaSum_336"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1831/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1831"
  op: "Mul"
  input: "Mul_1831/x"
  input: "Square_336"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1147"
  op: "AddV2"
  input: "Mul_1830"
  input: "Mul_1831"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_336"
  op: "Sqrt"
  input: "add_1147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1148/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1148"
  op: "AddV2"
  input: "Sqrt_336"
  input: "add_1148/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_339"
  op: "RealDiv"
  input: "add_1146"
  input: "add_1148"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1470"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1832/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1832"
  op: "Mul"
  input: "mul_1832/x"
  input: "ReadVariableOp_1470"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1149"
  op: "AddV2"
  input: "truediv_339"
  input: "mul_1832"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1833"
  op: "Mul"
  input: "mul_21"
  input: "add_1149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1471"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_340"
  op: "Sub"
  input: "ReadVariableOp_1471"
  input: "mul_1833"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1008"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_340"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1472"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1008"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1009"
  op: "AssignVariableOp"
  input: "Mul_1828/ReadVariableOp/Enter"
  input: "add_1146"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1473"
  op: "ReadVariableOp"
  input: "Mul_1828/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1009"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1010"
  op: "AssignVariableOp"
  input: "Mul_1830/ReadVariableOp/Enter"
  input: "add_1147"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1474"
  op: "ReadVariableOp"
  input: "Mul_1830/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1010"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1834/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1834/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1834/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1834/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1834"
  op: "Mul"
  input: "Mul_1834/x"
  input: "Mul_1834/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1835/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1835"
  op: "Mul"
  input: "Mul_1835/x"
  input: "CrossReplicaSum_337"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1150"
  op: "AddV2"
  input: "Mul_1834"
  input: "Mul_1835"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1836/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1836/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1836/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1836/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1836"
  op: "Mul"
  input: "Mul_1836/x"
  input: "Mul_1836/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_337"
  op: "Square"
  input: "CrossReplicaSum_337"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1837/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1837"
  op: "Mul"
  input: "Mul_1837/x"
  input: "Square_337"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1151"
  op: "AddV2"
  input: "Mul_1836"
  input: "Mul_1837"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_337"
  op: "Sqrt"
  input: "add_1151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1152/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1152"
  op: "AddV2"
  input: "Sqrt_337"
  input: "add_1152/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_340"
  op: "RealDiv"
  input: "add_1150"
  input: "add_1152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1838"
  op: "Mul"
  input: "mul_21"
  input: "truediv_340"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1475"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_341"
  op: "Sub"
  input: "ReadVariableOp_1475"
  input: "mul_1838"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1011"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_341"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1476"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1011"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1012"
  op: "AssignVariableOp"
  input: "Mul_1834/ReadVariableOp/Enter"
  input: "add_1150"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1477"
  op: "ReadVariableOp"
  input: "Mul_1834/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1012"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1013"
  op: "AssignVariableOp"
  input: "Mul_1836/ReadVariableOp/Enter"
  input: "add_1151"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1478"
  op: "ReadVariableOp"
  input: "Mul_1836/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1013"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1839/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1839/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1839/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1839/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1839"
  op: "Mul"
  input: "Mul_1839/x"
  input: "Mul_1839/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1840/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1840"
  op: "Mul"
  input: "Mul_1840/x"
  input: "CrossReplicaSum_338"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1153"
  op: "AddV2"
  input: "Mul_1839"
  input: "Mul_1840"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1841/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1841/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1841/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1841/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1841"
  op: "Mul"
  input: "Mul_1841/x"
  input: "Mul_1841/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_338"
  op: "Square"
  input: "CrossReplicaSum_338"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1842/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1842"
  op: "Mul"
  input: "Mul_1842/x"
  input: "Square_338"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1154"
  op: "AddV2"
  input: "Mul_1841"
  input: "Mul_1842"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_338"
  op: "Sqrt"
  input: "add_1154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1155/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1155"
  op: "AddV2"
  input: "Sqrt_338"
  input: "add_1155/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_341"
  op: "RealDiv"
  input: "add_1153"
  input: "add_1155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1479"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1843/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1843"
  op: "Mul"
  input: "mul_1843/x"
  input: "ReadVariableOp_1479"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1156"
  op: "AddV2"
  input: "truediv_341"
  input: "mul_1843"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1844"
  op: "Mul"
  input: "mul_21"
  input: "add_1156"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1480"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_342"
  op: "Sub"
  input: "ReadVariableOp_1480"
  input: "mul_1844"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1014"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_342"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1481"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1014"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1015"
  op: "AssignVariableOp"
  input: "Mul_1839/ReadVariableOp/Enter"
  input: "add_1153"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1482"
  op: "ReadVariableOp"
  input: "Mul_1839/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1015"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1016"
  op: "AssignVariableOp"
  input: "Mul_1841/ReadVariableOp/Enter"
  input: "add_1154"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1483"
  op: "ReadVariableOp"
  input: "Mul_1841/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1016"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1845/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1845/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1845/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1845/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1845"
  op: "Mul"
  input: "Mul_1845/x"
  input: "Mul_1845/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1846/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1846"
  op: "Mul"
  input: "Mul_1846/x"
  input: "CrossReplicaSum_339"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1157"
  op: "AddV2"
  input: "Mul_1845"
  input: "Mul_1846"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1847/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1847/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1847/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1847/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1847"
  op: "Mul"
  input: "Mul_1847/x"
  input: "Mul_1847/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_339"
  op: "Square"
  input: "CrossReplicaSum_339"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1848/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1848"
  op: "Mul"
  input: "Mul_1848/x"
  input: "Square_339"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1158"
  op: "AddV2"
  input: "Mul_1847"
  input: "Mul_1848"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_339"
  op: "Sqrt"
  input: "add_1158"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1159/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1159"
  op: "AddV2"
  input: "Sqrt_339"
  input: "add_1159/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_342"
  op: "RealDiv"
  input: "add_1157"
  input: "add_1159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1849"
  op: "Mul"
  input: "mul_21"
  input: "truediv_342"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1484"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_343"
  op: "Sub"
  input: "ReadVariableOp_1484"
  input: "mul_1849"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1017"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_343"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1485"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1017"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1018"
  op: "AssignVariableOp"
  input: "Mul_1845/ReadVariableOp/Enter"
  input: "add_1157"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1486"
  op: "ReadVariableOp"
  input: "Mul_1845/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1018"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1019"
  op: "AssignVariableOp"
  input: "Mul_1847/ReadVariableOp/Enter"
  input: "add_1158"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1487"
  op: "ReadVariableOp"
  input: "Mul_1847/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1019"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1850/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1850/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1850/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1850/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1850"
  op: "Mul"
  input: "Mul_1850/x"
  input: "Mul_1850/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1851/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1851"
  op: "Mul"
  input: "Mul_1851/x"
  input: "CrossReplicaSum_340"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1160"
  op: "AddV2"
  input: "Mul_1850"
  input: "Mul_1851"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1852/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1852/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1852/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1852/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1852"
  op: "Mul"
  input: "Mul_1852/x"
  input: "Mul_1852/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_340"
  op: "Square"
  input: "CrossReplicaSum_340"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1853/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1853"
  op: "Mul"
  input: "Mul_1853/x"
  input: "Square_340"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1161"
  op: "AddV2"
  input: "Mul_1852"
  input: "Mul_1853"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_340"
  op: "Sqrt"
  input: "add_1161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1162/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1162"
  op: "AddV2"
  input: "Sqrt_340"
  input: "add_1162/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_343"
  op: "RealDiv"
  input: "add_1160"
  input: "add_1162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1488"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1854/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1854"
  op: "Mul"
  input: "mul_1854/x"
  input: "ReadVariableOp_1488"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1163"
  op: "AddV2"
  input: "truediv_343"
  input: "mul_1854"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1855"
  op: "Mul"
  input: "mul_21"
  input: "add_1163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1489"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_344"
  op: "Sub"
  input: "ReadVariableOp_1489"
  input: "mul_1855"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1020"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_344"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1490"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1020"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1021"
  op: "AssignVariableOp"
  input: "Mul_1850/ReadVariableOp/Enter"
  input: "add_1160"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1491"
  op: "ReadVariableOp"
  input: "Mul_1850/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1021"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1022"
  op: "AssignVariableOp"
  input: "Mul_1852/ReadVariableOp/Enter"
  input: "add_1161"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1492"
  op: "ReadVariableOp"
  input: "Mul_1852/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1022"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1856/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1856/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1856/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1856/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1856"
  op: "Mul"
  input: "Mul_1856/x"
  input: "Mul_1856/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1857/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1857"
  op: "Mul"
  input: "Mul_1857/x"
  input: "CrossReplicaSum_341"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1164"
  op: "AddV2"
  input: "Mul_1856"
  input: "Mul_1857"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1858/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1858/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1858/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1858/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1858"
  op: "Mul"
  input: "Mul_1858/x"
  input: "Mul_1858/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_341"
  op: "Square"
  input: "CrossReplicaSum_341"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1859/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1859"
  op: "Mul"
  input: "Mul_1859/x"
  input: "Square_341"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1165"
  op: "AddV2"
  input: "Mul_1858"
  input: "Mul_1859"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_341"
  op: "Sqrt"
  input: "add_1165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1166/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1166"
  op: "AddV2"
  input: "Sqrt_341"
  input: "add_1166/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_344"
  op: "RealDiv"
  input: "add_1164"
  input: "add_1166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1860"
  op: "Mul"
  input: "mul_21"
  input: "truediv_344"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1493"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_345"
  op: "Sub"
  input: "ReadVariableOp_1493"
  input: "mul_1860"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1023"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_345"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1494"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1023"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1024"
  op: "AssignVariableOp"
  input: "Mul_1856/ReadVariableOp/Enter"
  input: "add_1164"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1495"
  op: "ReadVariableOp"
  input: "Mul_1856/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1024"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1025"
  op: "AssignVariableOp"
  input: "Mul_1858/ReadVariableOp/Enter"
  input: "add_1165"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1496"
  op: "ReadVariableOp"
  input: "Mul_1858/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1025"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1861/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1861/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1861/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1861/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1861"
  op: "Mul"
  input: "Mul_1861/x"
  input: "Mul_1861/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1862/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1862"
  op: "Mul"
  input: "Mul_1862/x"
  input: "CrossReplicaSum_342"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1167"
  op: "AddV2"
  input: "Mul_1861"
  input: "Mul_1862"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1863/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1863/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1863/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1863/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1863"
  op: "Mul"
  input: "Mul_1863/x"
  input: "Mul_1863/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_342"
  op: "Square"
  input: "CrossReplicaSum_342"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1864/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1864"
  op: "Mul"
  input: "Mul_1864/x"
  input: "Square_342"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1168"
  op: "AddV2"
  input: "Mul_1863"
  input: "Mul_1864"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_342"
  op: "Sqrt"
  input: "add_1168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1169/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1169"
  op: "AddV2"
  input: "Sqrt_342"
  input: "add_1169/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_345"
  op: "RealDiv"
  input: "add_1167"
  input: "add_1169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1865"
  op: "Mul"
  input: "mul_21"
  input: "truediv_345"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1497"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_346"
  op: "Sub"
  input: "ReadVariableOp_1497"
  input: "mul_1865"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1026"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_346"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1498"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1026"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1027"
  op: "AssignVariableOp"
  input: "Mul_1861/ReadVariableOp/Enter"
  input: "add_1167"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1499"
  op: "ReadVariableOp"
  input: "Mul_1861/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1027"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1028"
  op: "AssignVariableOp"
  input: "Mul_1863/ReadVariableOp/Enter"
  input: "add_1168"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1500"
  op: "ReadVariableOp"
  input: "Mul_1863/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1028"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1866/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1866/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1866/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1866/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1866"
  op: "Mul"
  input: "Mul_1866/x"
  input: "Mul_1866/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1867/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1867"
  op: "Mul"
  input: "Mul_1867/x"
  input: "CrossReplicaSum_343"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1170"
  op: "AddV2"
  input: "Mul_1866"
  input: "Mul_1867"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1868/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1868/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1868/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1868/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1868"
  op: "Mul"
  input: "Mul_1868/x"
  input: "Mul_1868/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_343"
  op: "Square"
  input: "CrossReplicaSum_343"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1869/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1869"
  op: "Mul"
  input: "Mul_1869/x"
  input: "Square_343"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1171"
  op: "AddV2"
  input: "Mul_1868"
  input: "Mul_1869"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_343"
  op: "Sqrt"
  input: "add_1171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1172/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1172"
  op: "AddV2"
  input: "Sqrt_343"
  input: "add_1172/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_346"
  op: "RealDiv"
  input: "add_1170"
  input: "add_1172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1870"
  op: "Mul"
  input: "mul_21"
  input: "truediv_346"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1501"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_347"
  op: "Sub"
  input: "ReadVariableOp_1501"
  input: "mul_1870"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1029"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_347"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1502"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1029"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1030"
  op: "AssignVariableOp"
  input: "Mul_1866/ReadVariableOp/Enter"
  input: "add_1170"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1503"
  op: "ReadVariableOp"
  input: "Mul_1866/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1030"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1031"
  op: "AssignVariableOp"
  input: "Mul_1868/ReadVariableOp/Enter"
  input: "add_1171"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1504"
  op: "ReadVariableOp"
  input: "Mul_1868/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1031"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1871/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1871/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1871/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1871/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1871"
  op: "Mul"
  input: "Mul_1871/x"
  input: "Mul_1871/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1872/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1872"
  op: "Mul"
  input: "Mul_1872/x"
  input: "CrossReplicaSum_344"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1173"
  op: "AddV2"
  input: "Mul_1871"
  input: "Mul_1872"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1873/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1873/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1873/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1873/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1873"
  op: "Mul"
  input: "Mul_1873/x"
  input: "Mul_1873/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_344"
  op: "Square"
  input: "CrossReplicaSum_344"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1874/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1874"
  op: "Mul"
  input: "Mul_1874/x"
  input: "Square_344"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1174"
  op: "AddV2"
  input: "Mul_1873"
  input: "Mul_1874"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_344"
  op: "Sqrt"
  input: "add_1174"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1175/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1175"
  op: "AddV2"
  input: "Sqrt_344"
  input: "add_1175/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_347"
  op: "RealDiv"
  input: "add_1173"
  input: "add_1175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1505"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1875/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1875"
  op: "Mul"
  input: "mul_1875/x"
  input: "ReadVariableOp_1505"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1176"
  op: "AddV2"
  input: "truediv_347"
  input: "mul_1875"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1876"
  op: "Mul"
  input: "mul_21"
  input: "add_1176"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1506"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_348"
  op: "Sub"
  input: "ReadVariableOp_1506"
  input: "mul_1876"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1032"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_348"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1507"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1032"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1033"
  op: "AssignVariableOp"
  input: "Mul_1871/ReadVariableOp/Enter"
  input: "add_1173"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1508"
  op: "ReadVariableOp"
  input: "Mul_1871/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1033"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1034"
  op: "AssignVariableOp"
  input: "Mul_1873/ReadVariableOp/Enter"
  input: "add_1174"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1509"
  op: "ReadVariableOp"
  input: "Mul_1873/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1034"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1877/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1877/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1877/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1877/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1877"
  op: "Mul"
  input: "Mul_1877/x"
  input: "Mul_1877/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1878/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1878"
  op: "Mul"
  input: "Mul_1878/x"
  input: "CrossReplicaSum_345"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1177"
  op: "AddV2"
  input: "Mul_1877"
  input: "Mul_1878"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1879/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1879/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1879/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1879/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1879"
  op: "Mul"
  input: "Mul_1879/x"
  input: "Mul_1879/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_345"
  op: "Square"
  input: "CrossReplicaSum_345"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1880/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1880"
  op: "Mul"
  input: "Mul_1880/x"
  input: "Square_345"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1178"
  op: "AddV2"
  input: "Mul_1879"
  input: "Mul_1880"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_345"
  op: "Sqrt"
  input: "add_1178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1179/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1179"
  op: "AddV2"
  input: "Sqrt_345"
  input: "add_1179/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_348"
  op: "RealDiv"
  input: "add_1177"
  input: "add_1179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1881"
  op: "Mul"
  input: "mul_21"
  input: "truediv_348"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1510"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_349"
  op: "Sub"
  input: "ReadVariableOp_1510"
  input: "mul_1881"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1035"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_349"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1511"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1035"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1036"
  op: "AssignVariableOp"
  input: "Mul_1877/ReadVariableOp/Enter"
  input: "add_1177"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1512"
  op: "ReadVariableOp"
  input: "Mul_1877/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1036"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1037"
  op: "AssignVariableOp"
  input: "Mul_1879/ReadVariableOp/Enter"
  input: "add_1178"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1513"
  op: "ReadVariableOp"
  input: "Mul_1879/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1037"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1882/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1882/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1882/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1882/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1882"
  op: "Mul"
  input: "Mul_1882/x"
  input: "Mul_1882/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1883/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1883"
  op: "Mul"
  input: "Mul_1883/x"
  input: "CrossReplicaSum_346"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1180"
  op: "AddV2"
  input: "Mul_1882"
  input: "Mul_1883"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1884/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1884/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1884/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1884/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1884"
  op: "Mul"
  input: "Mul_1884/x"
  input: "Mul_1884/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_346"
  op: "Square"
  input: "CrossReplicaSum_346"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1885/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1885"
  op: "Mul"
  input: "Mul_1885/x"
  input: "Square_346"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1181"
  op: "AddV2"
  input: "Mul_1884"
  input: "Mul_1885"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_346"
  op: "Sqrt"
  input: "add_1181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1182/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1182"
  op: "AddV2"
  input: "Sqrt_346"
  input: "add_1182/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_349"
  op: "RealDiv"
  input: "add_1180"
  input: "add_1182"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1514"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1886/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1886"
  op: "Mul"
  input: "mul_1886/x"
  input: "ReadVariableOp_1514"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1183"
  op: "AddV2"
  input: "truediv_349"
  input: "mul_1886"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1887"
  op: "Mul"
  input: "mul_21"
  input: "add_1183"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1515"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_350"
  op: "Sub"
  input: "ReadVariableOp_1515"
  input: "mul_1887"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1038"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_350"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1516"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1038"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1039"
  op: "AssignVariableOp"
  input: "Mul_1882/ReadVariableOp/Enter"
  input: "add_1180"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1517"
  op: "ReadVariableOp"
  input: "Mul_1882/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1039"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1040"
  op: "AssignVariableOp"
  input: "Mul_1884/ReadVariableOp/Enter"
  input: "add_1181"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1518"
  op: "ReadVariableOp"
  input: "Mul_1884/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1040"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_m"
  input: "electra/encoder/layer_8/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_v"
  input: "electra/encoder/layer_8/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1888/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1888/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1888/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1888/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1888"
  op: "Mul"
  input: "Mul_1888/x"
  input: "Mul_1888/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1889/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1889"
  op: "Mul"
  input: "Mul_1889/x"
  input: "CrossReplicaSum_347"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1184"
  op: "AddV2"
  input: "Mul_1888"
  input: "Mul_1889"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1890/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1890/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1890/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1890/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1890"
  op: "Mul"
  input: "Mul_1890/x"
  input: "Mul_1890/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_347"
  op: "Square"
  input: "CrossReplicaSum_347"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1891/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1891"
  op: "Mul"
  input: "Mul_1891/x"
  input: "Square_347"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1185"
  op: "AddV2"
  input: "Mul_1890"
  input: "Mul_1891"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_347"
  op: "Sqrt"
  input: "add_1185"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1186/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1186"
  op: "AddV2"
  input: "Sqrt_347"
  input: "add_1186/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_350"
  op: "RealDiv"
  input: "add_1184"
  input: "add_1186"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1892"
  op: "Mul"
  input: "mul_21"
  input: "truediv_350"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1519"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_351"
  op: "Sub"
  input: "ReadVariableOp_1519"
  input: "mul_1892"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1041"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_351"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1520"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1041"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1042"
  op: "AssignVariableOp"
  input: "Mul_1888/ReadVariableOp/Enter"
  input: "add_1184"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1521"
  op: "ReadVariableOp"
  input: "Mul_1888/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1042"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1043"
  op: "AssignVariableOp"
  input: "Mul_1890/ReadVariableOp/Enter"
  input: "add_1185"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1522"
  op: "ReadVariableOp"
  input: "Mul_1890/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1043"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1893/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1893/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1893/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1893/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1893"
  op: "Mul"
  input: "Mul_1893/x"
  input: "Mul_1893/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1894/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1894"
  op: "Mul"
  input: "Mul_1894/x"
  input: "CrossReplicaSum_348"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1187"
  op: "AddV2"
  input: "Mul_1893"
  input: "Mul_1894"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1895/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1895/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1895/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1895/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1895"
  op: "Mul"
  input: "Mul_1895/x"
  input: "Mul_1895/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_348"
  op: "Square"
  input: "CrossReplicaSum_348"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1896/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1896"
  op: "Mul"
  input: "Mul_1896/x"
  input: "Square_348"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1188"
  op: "AddV2"
  input: "Mul_1895"
  input: "Mul_1896"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_348"
  op: "Sqrt"
  input: "add_1188"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1189/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1189"
  op: "AddV2"
  input: "Sqrt_348"
  input: "add_1189/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_351"
  op: "RealDiv"
  input: "add_1187"
  input: "add_1189"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1897"
  op: "Mul"
  input: "mul_21"
  input: "truediv_351"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1523"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_352"
  op: "Sub"
  input: "ReadVariableOp_1523"
  input: "mul_1897"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1044"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_352"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1524"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1044"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1045"
  op: "AssignVariableOp"
  input: "Mul_1893/ReadVariableOp/Enter"
  input: "add_1187"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1525"
  op: "ReadVariableOp"
  input: "Mul_1893/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1045"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1046"
  op: "AssignVariableOp"
  input: "Mul_1895/ReadVariableOp/Enter"
  input: "add_1188"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1526"
  op: "ReadVariableOp"
  input: "Mul_1895/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1046"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1898/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1898/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1898/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1898/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1898"
  op: "Mul"
  input: "Mul_1898/x"
  input: "Mul_1898/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1899/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1899"
  op: "Mul"
  input: "Mul_1899/x"
  input: "CrossReplicaSum_349"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1190"
  op: "AddV2"
  input: "Mul_1898"
  input: "Mul_1899"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1900/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1900/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1900/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1900/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1900"
  op: "Mul"
  input: "Mul_1900/x"
  input: "Mul_1900/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_349"
  op: "Square"
  input: "CrossReplicaSum_349"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1901/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1901"
  op: "Mul"
  input: "Mul_1901/x"
  input: "Square_349"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1191"
  op: "AddV2"
  input: "Mul_1900"
  input: "Mul_1901"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_349"
  op: "Sqrt"
  input: "add_1191"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1192/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1192"
  op: "AddV2"
  input: "Sqrt_349"
  input: "add_1192/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_352"
  op: "RealDiv"
  input: "add_1190"
  input: "add_1192"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1902"
  op: "Mul"
  input: "mul_21"
  input: "truediv_352"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1527"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_353"
  op: "Sub"
  input: "ReadVariableOp_1527"
  input: "mul_1902"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1047"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_353"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1528"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1047"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1048"
  op: "AssignVariableOp"
  input: "Mul_1898/ReadVariableOp/Enter"
  input: "add_1190"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1529"
  op: "ReadVariableOp"
  input: "Mul_1898/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1048"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1049"
  op: "AssignVariableOp"
  input: "Mul_1900/ReadVariableOp/Enter"
  input: "add_1191"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1530"
  op: "ReadVariableOp"
  input: "Mul_1900/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1049"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1903/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1903/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1903/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1903/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1903"
  op: "Mul"
  input: "Mul_1903/x"
  input: "Mul_1903/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1904/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1904"
  op: "Mul"
  input: "Mul_1904/x"
  input: "CrossReplicaSum_350"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1193"
  op: "AddV2"
  input: "Mul_1903"
  input: "Mul_1904"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1905/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1905/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1905/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1905/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1905"
  op: "Mul"
  input: "Mul_1905/x"
  input: "Mul_1905/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_350"
  op: "Square"
  input: "CrossReplicaSum_350"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1906/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1906"
  op: "Mul"
  input: "Mul_1906/x"
  input: "Square_350"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1194"
  op: "AddV2"
  input: "Mul_1905"
  input: "Mul_1906"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_350"
  op: "Sqrt"
  input: "add_1194"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1195/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1195"
  op: "AddV2"
  input: "Sqrt_350"
  input: "add_1195/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_353"
  op: "RealDiv"
  input: "add_1193"
  input: "add_1195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1531"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1907/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1907"
  op: "Mul"
  input: "mul_1907/x"
  input: "ReadVariableOp_1531"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1196"
  op: "AddV2"
  input: "truediv_353"
  input: "mul_1907"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1908"
  op: "Mul"
  input: "mul_21"
  input: "add_1196"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1532"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_354"
  op: "Sub"
  input: "ReadVariableOp_1532"
  input: "mul_1908"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1050"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_354"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1533"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1050"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1051"
  op: "AssignVariableOp"
  input: "Mul_1903/ReadVariableOp/Enter"
  input: "add_1193"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1534"
  op: "ReadVariableOp"
  input: "Mul_1903/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1051"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1052"
  op: "AssignVariableOp"
  input: "Mul_1905/ReadVariableOp/Enter"
  input: "add_1194"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1535"
  op: "ReadVariableOp"
  input: "Mul_1905/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1052"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1909/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1909/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1909/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1909/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1909"
  op: "Mul"
  input: "Mul_1909/x"
  input: "Mul_1909/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1910/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1910"
  op: "Mul"
  input: "Mul_1910/x"
  input: "CrossReplicaSum_351"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1197"
  op: "AddV2"
  input: "Mul_1909"
  input: "Mul_1910"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1911/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1911/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1911/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1911/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1911"
  op: "Mul"
  input: "Mul_1911/x"
  input: "Mul_1911/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_351"
  op: "Square"
  input: "CrossReplicaSum_351"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1912/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1912"
  op: "Mul"
  input: "Mul_1912/x"
  input: "Square_351"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1198"
  op: "AddV2"
  input: "Mul_1911"
  input: "Mul_1912"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_351"
  op: "Sqrt"
  input: "add_1198"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1199/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1199"
  op: "AddV2"
  input: "Sqrt_351"
  input: "add_1199/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_354"
  op: "RealDiv"
  input: "add_1197"
  input: "add_1199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1913"
  op: "Mul"
  input: "mul_21"
  input: "truediv_354"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1536"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_355"
  op: "Sub"
  input: "ReadVariableOp_1536"
  input: "mul_1913"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1053"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_355"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1537"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1053"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1054"
  op: "AssignVariableOp"
  input: "Mul_1909/ReadVariableOp/Enter"
  input: "add_1197"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1538"
  op: "ReadVariableOp"
  input: "Mul_1909/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1054"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1055"
  op: "AssignVariableOp"
  input: "Mul_1911/ReadVariableOp/Enter"
  input: "add_1198"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1539"
  op: "ReadVariableOp"
  input: "Mul_1911/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1055"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1914/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1914/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1914/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1914/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1914"
  op: "Mul"
  input: "Mul_1914/x"
  input: "Mul_1914/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1915/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1915"
  op: "Mul"
  input: "Mul_1915/x"
  input: "CrossReplicaSum_352"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1200"
  op: "AddV2"
  input: "Mul_1914"
  input: "Mul_1915"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1916/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1916/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1916/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1916/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1916"
  op: "Mul"
  input: "Mul_1916/x"
  input: "Mul_1916/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_352"
  op: "Square"
  input: "CrossReplicaSum_352"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1917/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1917"
  op: "Mul"
  input: "Mul_1917/x"
  input: "Square_352"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1201"
  op: "AddV2"
  input: "Mul_1916"
  input: "Mul_1917"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_352"
  op: "Sqrt"
  input: "add_1201"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1202/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1202"
  op: "AddV2"
  input: "Sqrt_352"
  input: "add_1202/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_355"
  op: "RealDiv"
  input: "add_1200"
  input: "add_1202"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1540"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1918/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1918"
  op: "Mul"
  input: "mul_1918/x"
  input: "ReadVariableOp_1540"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1203"
  op: "AddV2"
  input: "truediv_355"
  input: "mul_1918"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1919"
  op: "Mul"
  input: "mul_21"
  input: "add_1203"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1541"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_356"
  op: "Sub"
  input: "ReadVariableOp_1541"
  input: "mul_1919"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1056"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_356"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1542"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1056"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1057"
  op: "AssignVariableOp"
  input: "Mul_1914/ReadVariableOp/Enter"
  input: "add_1200"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1543"
  op: "ReadVariableOp"
  input: "Mul_1914/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1057"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1058"
  op: "AssignVariableOp"
  input: "Mul_1916/ReadVariableOp/Enter"
  input: "add_1201"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1544"
  op: "ReadVariableOp"
  input: "Mul_1916/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1058"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1920/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1920/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1920/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1920/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1920"
  op: "Mul"
  input: "Mul_1920/x"
  input: "Mul_1920/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1921/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1921"
  op: "Mul"
  input: "Mul_1921/x"
  input: "CrossReplicaSum_353"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1204"
  op: "AddV2"
  input: "Mul_1920"
  input: "Mul_1921"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1922/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1922/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1922/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1922/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1922"
  op: "Mul"
  input: "Mul_1922/x"
  input: "Mul_1922/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_353"
  op: "Square"
  input: "CrossReplicaSum_353"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1923/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1923"
  op: "Mul"
  input: "Mul_1923/x"
  input: "Square_353"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1205"
  op: "AddV2"
  input: "Mul_1922"
  input: "Mul_1923"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_353"
  op: "Sqrt"
  input: "add_1205"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1206/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1206"
  op: "AddV2"
  input: "Sqrt_353"
  input: "add_1206/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_356"
  op: "RealDiv"
  input: "add_1204"
  input: "add_1206"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1924"
  op: "Mul"
  input: "mul_21"
  input: "truediv_356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1545"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_357"
  op: "Sub"
  input: "ReadVariableOp_1545"
  input: "mul_1924"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1059"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_357"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1546"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1059"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1060"
  op: "AssignVariableOp"
  input: "Mul_1920/ReadVariableOp/Enter"
  input: "add_1204"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1547"
  op: "ReadVariableOp"
  input: "Mul_1920/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1060"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1061"
  op: "AssignVariableOp"
  input: "Mul_1922/ReadVariableOp/Enter"
  input: "add_1205"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1548"
  op: "ReadVariableOp"
  input: "Mul_1922/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1061"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1925/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1925/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1925/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1925/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1925"
  op: "Mul"
  input: "Mul_1925/x"
  input: "Mul_1925/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1926/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1926"
  op: "Mul"
  input: "Mul_1926/x"
  input: "CrossReplicaSum_354"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1207"
  op: "AddV2"
  input: "Mul_1925"
  input: "Mul_1926"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1927/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1927/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1927/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1927/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1927"
  op: "Mul"
  input: "Mul_1927/x"
  input: "Mul_1927/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_354"
  op: "Square"
  input: "CrossReplicaSum_354"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1928/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1928"
  op: "Mul"
  input: "Mul_1928/x"
  input: "Square_354"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1208"
  op: "AddV2"
  input: "Mul_1927"
  input: "Mul_1928"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_354"
  op: "Sqrt"
  input: "add_1208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1209/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1209"
  op: "AddV2"
  input: "Sqrt_354"
  input: "add_1209/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_357"
  op: "RealDiv"
  input: "add_1207"
  input: "add_1209"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1549"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1929/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1929"
  op: "Mul"
  input: "mul_1929/x"
  input: "ReadVariableOp_1549"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1210"
  op: "AddV2"
  input: "truediv_357"
  input: "mul_1929"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1930"
  op: "Mul"
  input: "mul_21"
  input: "add_1210"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1550"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_358"
  op: "Sub"
  input: "ReadVariableOp_1550"
  input: "mul_1930"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1062"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_358"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1551"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1062"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1063"
  op: "AssignVariableOp"
  input: "Mul_1925/ReadVariableOp/Enter"
  input: "add_1207"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1552"
  op: "ReadVariableOp"
  input: "Mul_1925/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1063"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1064"
  op: "AssignVariableOp"
  input: "Mul_1927/ReadVariableOp/Enter"
  input: "add_1208"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1553"
  op: "ReadVariableOp"
  input: "Mul_1927/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1064"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1931/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1931/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1931/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1931/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1931"
  op: "Mul"
  input: "Mul_1931/x"
  input: "Mul_1931/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1932/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1932"
  op: "Mul"
  input: "Mul_1932/x"
  input: "CrossReplicaSum_355"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1211"
  op: "AddV2"
  input: "Mul_1931"
  input: "Mul_1932"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1933/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1933/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1933/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1933/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1933"
  op: "Mul"
  input: "Mul_1933/x"
  input: "Mul_1933/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_355"
  op: "Square"
  input: "CrossReplicaSum_355"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1934/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1934"
  op: "Mul"
  input: "Mul_1934/x"
  input: "Square_355"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1212"
  op: "AddV2"
  input: "Mul_1933"
  input: "Mul_1934"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_355"
  op: "Sqrt"
  input: "add_1212"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1213/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1213"
  op: "AddV2"
  input: "Sqrt_355"
  input: "add_1213/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_358"
  op: "RealDiv"
  input: "add_1211"
  input: "add_1213"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1935"
  op: "Mul"
  input: "mul_21"
  input: "truediv_358"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1554"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_359"
  op: "Sub"
  input: "ReadVariableOp_1554"
  input: "mul_1935"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1065"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_359"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1555"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1065"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1066"
  op: "AssignVariableOp"
  input: "Mul_1931/ReadVariableOp/Enter"
  input: "add_1211"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1556"
  op: "ReadVariableOp"
  input: "Mul_1931/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1066"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1067"
  op: "AssignVariableOp"
  input: "Mul_1933/ReadVariableOp/Enter"
  input: "add_1212"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1557"
  op: "ReadVariableOp"
  input: "Mul_1933/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1067"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1936/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1936/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1936/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1936/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1936"
  op: "Mul"
  input: "Mul_1936/x"
  input: "Mul_1936/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1937/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1937"
  op: "Mul"
  input: "Mul_1937/x"
  input: "CrossReplicaSum_356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1214"
  op: "AddV2"
  input: "Mul_1936"
  input: "Mul_1937"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1938/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1938/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1938/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1938/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1938"
  op: "Mul"
  input: "Mul_1938/x"
  input: "Mul_1938/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_356"
  op: "Square"
  input: "CrossReplicaSum_356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1939/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1939"
  op: "Mul"
  input: "Mul_1939/x"
  input: "Square_356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1215"
  op: "AddV2"
  input: "Mul_1938"
  input: "Mul_1939"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_356"
  op: "Sqrt"
  input: "add_1215"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1216/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1216"
  op: "AddV2"
  input: "Sqrt_356"
  input: "add_1216/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_359"
  op: "RealDiv"
  input: "add_1214"
  input: "add_1216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1558"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1940/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1940"
  op: "Mul"
  input: "mul_1940/x"
  input: "ReadVariableOp_1558"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1217"
  op: "AddV2"
  input: "truediv_359"
  input: "mul_1940"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1941"
  op: "Mul"
  input: "mul_21"
  input: "add_1217"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1559"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_360"
  op: "Sub"
  input: "ReadVariableOp_1559"
  input: "mul_1941"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1068"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_360"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1560"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1068"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1069"
  op: "AssignVariableOp"
  input: "Mul_1936/ReadVariableOp/Enter"
  input: "add_1214"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1561"
  op: "ReadVariableOp"
  input: "Mul_1936/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1069"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1070"
  op: "AssignVariableOp"
  input: "Mul_1938/ReadVariableOp/Enter"
  input: "add_1215"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1562"
  op: "ReadVariableOp"
  input: "Mul_1938/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1070"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1942/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1942/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1942/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1942/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1942"
  op: "Mul"
  input: "Mul_1942/x"
  input: "Mul_1942/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1943/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1943"
  op: "Mul"
  input: "Mul_1943/x"
  input: "CrossReplicaSum_357"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1218"
  op: "AddV2"
  input: "Mul_1942"
  input: "Mul_1943"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1944/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1944/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1944/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1944/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1944"
  op: "Mul"
  input: "Mul_1944/x"
  input: "Mul_1944/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_357"
  op: "Square"
  input: "CrossReplicaSum_357"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1945/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1945"
  op: "Mul"
  input: "Mul_1945/x"
  input: "Square_357"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1219"
  op: "AddV2"
  input: "Mul_1944"
  input: "Mul_1945"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_357"
  op: "Sqrt"
  input: "add_1219"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1220/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1220"
  op: "AddV2"
  input: "Sqrt_357"
  input: "add_1220/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_360"
  op: "RealDiv"
  input: "add_1218"
  input: "add_1220"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1946"
  op: "Mul"
  input: "mul_21"
  input: "truediv_360"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1563"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_361"
  op: "Sub"
  input: "ReadVariableOp_1563"
  input: "mul_1946"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1071"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_361"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1564"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1071"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1072"
  op: "AssignVariableOp"
  input: "Mul_1942/ReadVariableOp/Enter"
  input: "add_1218"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1565"
  op: "ReadVariableOp"
  input: "Mul_1942/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1072"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1073"
  op: "AssignVariableOp"
  input: "Mul_1944/ReadVariableOp/Enter"
  input: "add_1219"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1566"
  op: "ReadVariableOp"
  input: "Mul_1944/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1073"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1947/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1947/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1947/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1947/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1947"
  op: "Mul"
  input: "Mul_1947/x"
  input: "Mul_1947/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1948/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1948"
  op: "Mul"
  input: "Mul_1948/x"
  input: "CrossReplicaSum_358"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1221"
  op: "AddV2"
  input: "Mul_1947"
  input: "Mul_1948"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1949/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1949/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1949/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1949/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1949"
  op: "Mul"
  input: "Mul_1949/x"
  input: "Mul_1949/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_358"
  op: "Square"
  input: "CrossReplicaSum_358"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1950/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1950"
  op: "Mul"
  input: "Mul_1950/x"
  input: "Square_358"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1222"
  op: "AddV2"
  input: "Mul_1949"
  input: "Mul_1950"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_358"
  op: "Sqrt"
  input: "add_1222"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1223/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1223"
  op: "AddV2"
  input: "Sqrt_358"
  input: "add_1223/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_361"
  op: "RealDiv"
  input: "add_1221"
  input: "add_1223"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1951"
  op: "Mul"
  input: "mul_21"
  input: "truediv_361"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1567"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_362"
  op: "Sub"
  input: "ReadVariableOp_1567"
  input: "mul_1951"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1074"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_362"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1568"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1074"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1075"
  op: "AssignVariableOp"
  input: "Mul_1947/ReadVariableOp/Enter"
  input: "add_1221"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1569"
  op: "ReadVariableOp"
  input: "Mul_1947/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1075"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1076"
  op: "AssignVariableOp"
  input: "Mul_1949/ReadVariableOp/Enter"
  input: "add_1222"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1570"
  op: "ReadVariableOp"
  input: "Mul_1949/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1076"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1952/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1952/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1952/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1952/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1952"
  op: "Mul"
  input: "Mul_1952/x"
  input: "Mul_1952/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1953/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1953"
  op: "Mul"
  input: "Mul_1953/x"
  input: "CrossReplicaSum_359"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1224"
  op: "AddV2"
  input: "Mul_1952"
  input: "Mul_1953"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1954/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1954/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1954/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1954/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1954"
  op: "Mul"
  input: "Mul_1954/x"
  input: "Mul_1954/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_359"
  op: "Square"
  input: "CrossReplicaSum_359"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1955/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1955"
  op: "Mul"
  input: "Mul_1955/x"
  input: "Square_359"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1225"
  op: "AddV2"
  input: "Mul_1954"
  input: "Mul_1955"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_359"
  op: "Sqrt"
  input: "add_1225"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1226/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1226"
  op: "AddV2"
  input: "Sqrt_359"
  input: "add_1226/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_362"
  op: "RealDiv"
  input: "add_1224"
  input: "add_1226"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1956"
  op: "Mul"
  input: "mul_21"
  input: "truediv_362"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1571"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_363"
  op: "Sub"
  input: "ReadVariableOp_1571"
  input: "mul_1956"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1077"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_363"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1572"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1077"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1078"
  op: "AssignVariableOp"
  input: "Mul_1952/ReadVariableOp/Enter"
  input: "add_1224"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1573"
  op: "ReadVariableOp"
  input: "Mul_1952/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1078"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1079"
  op: "AssignVariableOp"
  input: "Mul_1954/ReadVariableOp/Enter"
  input: "add_1225"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1574"
  op: "ReadVariableOp"
  input: "Mul_1954/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1079"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1957/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1957/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1957/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1957/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1957"
  op: "Mul"
  input: "Mul_1957/x"
  input: "Mul_1957/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1958/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1958"
  op: "Mul"
  input: "Mul_1958/x"
  input: "CrossReplicaSum_360"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1227"
  op: "AddV2"
  input: "Mul_1957"
  input: "Mul_1958"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1959/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1959/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1959/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1959/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1959"
  op: "Mul"
  input: "Mul_1959/x"
  input: "Mul_1959/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_360"
  op: "Square"
  input: "CrossReplicaSum_360"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1960/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1960"
  op: "Mul"
  input: "Mul_1960/x"
  input: "Square_360"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1228"
  op: "AddV2"
  input: "Mul_1959"
  input: "Mul_1960"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_360"
  op: "Sqrt"
  input: "add_1228"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1229/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1229"
  op: "AddV2"
  input: "Sqrt_360"
  input: "add_1229/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_363"
  op: "RealDiv"
  input: "add_1227"
  input: "add_1229"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1575"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1961/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1961"
  op: "Mul"
  input: "mul_1961/x"
  input: "ReadVariableOp_1575"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1230"
  op: "AddV2"
  input: "truediv_363"
  input: "mul_1961"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1962"
  op: "Mul"
  input: "mul_21"
  input: "add_1230"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1576"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_364"
  op: "Sub"
  input: "ReadVariableOp_1576"
  input: "mul_1962"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1080"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_364"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1577"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1080"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1081"
  op: "AssignVariableOp"
  input: "Mul_1957/ReadVariableOp/Enter"
  input: "add_1227"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1578"
  op: "ReadVariableOp"
  input: "Mul_1957/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1081"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1082"
  op: "AssignVariableOp"
  input: "Mul_1959/ReadVariableOp/Enter"
  input: "add_1228"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1579"
  op: "ReadVariableOp"
  input: "Mul_1959/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1082"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1963/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1963/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1963/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1963/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1963"
  op: "Mul"
  input: "Mul_1963/x"
  input: "Mul_1963/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1964/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1964"
  op: "Mul"
  input: "Mul_1964/x"
  input: "CrossReplicaSum_361"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1231"
  op: "AddV2"
  input: "Mul_1963"
  input: "Mul_1964"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1965/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1965/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1965/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1965/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1965"
  op: "Mul"
  input: "Mul_1965/x"
  input: "Mul_1965/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_361"
  op: "Square"
  input: "CrossReplicaSum_361"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1966/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1966"
  op: "Mul"
  input: "Mul_1966/x"
  input: "Square_361"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1232"
  op: "AddV2"
  input: "Mul_1965"
  input: "Mul_1966"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_361"
  op: "Sqrt"
  input: "add_1232"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1233/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1233"
  op: "AddV2"
  input: "Sqrt_361"
  input: "add_1233/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_364"
  op: "RealDiv"
  input: "add_1231"
  input: "add_1233"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1967"
  op: "Mul"
  input: "mul_21"
  input: "truediv_364"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1580"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_365"
  op: "Sub"
  input: "ReadVariableOp_1580"
  input: "mul_1967"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1083"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_365"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1581"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1083"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1084"
  op: "AssignVariableOp"
  input: "Mul_1963/ReadVariableOp/Enter"
  input: "add_1231"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1582"
  op: "ReadVariableOp"
  input: "Mul_1963/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1084"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1085"
  op: "AssignVariableOp"
  input: "Mul_1965/ReadVariableOp/Enter"
  input: "add_1232"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1583"
  op: "ReadVariableOp"
  input: "Mul_1965/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1085"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1968/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1968/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1968/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1968/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1968"
  op: "Mul"
  input: "Mul_1968/x"
  input: "Mul_1968/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1969/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1969"
  op: "Mul"
  input: "Mul_1969/x"
  input: "CrossReplicaSum_362"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1234"
  op: "AddV2"
  input: "Mul_1968"
  input: "Mul_1969"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1970/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1970/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1970/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1970/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1970"
  op: "Mul"
  input: "Mul_1970/x"
  input: "Mul_1970/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_362"
  op: "Square"
  input: "CrossReplicaSum_362"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1971/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1971"
  op: "Mul"
  input: "Mul_1971/x"
  input: "Square_362"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1235"
  op: "AddV2"
  input: "Mul_1970"
  input: "Mul_1971"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_362"
  op: "Sqrt"
  input: "add_1235"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1236/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1236"
  op: "AddV2"
  input: "Sqrt_362"
  input: "add_1236/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_365"
  op: "RealDiv"
  input: "add_1234"
  input: "add_1236"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1584"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1972/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1972"
  op: "Mul"
  input: "mul_1972/x"
  input: "ReadVariableOp_1584"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1237"
  op: "AddV2"
  input: "truediv_365"
  input: "mul_1972"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1973"
  op: "Mul"
  input: "mul_21"
  input: "add_1237"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1585"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_366"
  op: "Sub"
  input: "ReadVariableOp_1585"
  input: "mul_1973"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1086"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_366"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1586"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1086"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1087"
  op: "AssignVariableOp"
  input: "Mul_1968/ReadVariableOp/Enter"
  input: "add_1234"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1587"
  op: "ReadVariableOp"
  input: "Mul_1968/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1087"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1088"
  op: "AssignVariableOp"
  input: "Mul_1970/ReadVariableOp/Enter"
  input: "add_1235"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1588"
  op: "ReadVariableOp"
  input: "Mul_1970/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1088"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_m"
  input: "electra/encoder/layer_9/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_v"
  input: "electra/encoder/layer_9/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1974/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1974/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1974/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1974/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1974"
  op: "Mul"
  input: "Mul_1974/x"
  input: "Mul_1974/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1975/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1975"
  op: "Mul"
  input: "Mul_1975/x"
  input: "CrossReplicaSum_363"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1238"
  op: "AddV2"
  input: "Mul_1974"
  input: "Mul_1975"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1976/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1976/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1976/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1976/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1976"
  op: "Mul"
  input: "Mul_1976/x"
  input: "Mul_1976/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_363"
  op: "Square"
  input: "CrossReplicaSum_363"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1977/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1977"
  op: "Mul"
  input: "Mul_1977/x"
  input: "Square_363"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1239"
  op: "AddV2"
  input: "Mul_1976"
  input: "Mul_1977"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_363"
  op: "Sqrt"
  input: "add_1239"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1240/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1240"
  op: "AddV2"
  input: "Sqrt_363"
  input: "add_1240/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_366"
  op: "RealDiv"
  input: "add_1238"
  input: "add_1240"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1978"
  op: "Mul"
  input: "mul_21"
  input: "truediv_366"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1589"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_367"
  op: "Sub"
  input: "ReadVariableOp_1589"
  input: "mul_1978"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1089"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_367"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1590"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1089"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1090"
  op: "AssignVariableOp"
  input: "Mul_1974/ReadVariableOp/Enter"
  input: "add_1238"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1591"
  op: "ReadVariableOp"
  input: "Mul_1974/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1090"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1091"
  op: "AssignVariableOp"
  input: "Mul_1976/ReadVariableOp/Enter"
  input: "add_1239"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1592"
  op: "ReadVariableOp"
  input: "Mul_1976/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1091"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1979/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1979/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1979/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1979/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1979"
  op: "Mul"
  input: "Mul_1979/x"
  input: "Mul_1979/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1980/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1980"
  op: "Mul"
  input: "Mul_1980/x"
  input: "CrossReplicaSum_364"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1241"
  op: "AddV2"
  input: "Mul_1979"
  input: "Mul_1980"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1981/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1981/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1981/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1981/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1981"
  op: "Mul"
  input: "Mul_1981/x"
  input: "Mul_1981/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_364"
  op: "Square"
  input: "CrossReplicaSum_364"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1982/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1982"
  op: "Mul"
  input: "Mul_1982/x"
  input: "Square_364"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1242"
  op: "AddV2"
  input: "Mul_1981"
  input: "Mul_1982"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_364"
  op: "Sqrt"
  input: "add_1242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1243/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1243"
  op: "AddV2"
  input: "Sqrt_364"
  input: "add_1243/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_367"
  op: "RealDiv"
  input: "add_1241"
  input: "add_1243"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1983"
  op: "Mul"
  input: "mul_21"
  input: "truediv_367"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1593"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_368"
  op: "Sub"
  input: "ReadVariableOp_1593"
  input: "mul_1983"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1092"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_368"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1594"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1092"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1093"
  op: "AssignVariableOp"
  input: "Mul_1979/ReadVariableOp/Enter"
  input: "add_1241"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1595"
  op: "ReadVariableOp"
  input: "Mul_1979/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1093"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1094"
  op: "AssignVariableOp"
  input: "Mul_1981/ReadVariableOp/Enter"
  input: "add_1242"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1596"
  op: "ReadVariableOp"
  input: "Mul_1981/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1094"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1984/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1984/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1984/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1984/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1984"
  op: "Mul"
  input: "Mul_1984/x"
  input: "Mul_1984/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1985/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1985"
  op: "Mul"
  input: "Mul_1985/x"
  input: "CrossReplicaSum_365"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1244"
  op: "AddV2"
  input: "Mul_1984"
  input: "Mul_1985"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1986/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1986/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1986/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1986/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1986"
  op: "Mul"
  input: "Mul_1986/x"
  input: "Mul_1986/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_365"
  op: "Square"
  input: "CrossReplicaSum_365"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1987/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1987"
  op: "Mul"
  input: "Mul_1987/x"
  input: "Square_365"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1245"
  op: "AddV2"
  input: "Mul_1986"
  input: "Mul_1987"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_365"
  op: "Sqrt"
  input: "add_1245"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1246/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1246"
  op: "AddV2"
  input: "Sqrt_365"
  input: "add_1246/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_368"
  op: "RealDiv"
  input: "add_1244"
  input: "add_1246"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1988"
  op: "Mul"
  input: "mul_21"
  input: "truediv_368"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1597"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_369"
  op: "Sub"
  input: "ReadVariableOp_1597"
  input: "mul_1988"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1095"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_369"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1598"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1095"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1096"
  op: "AssignVariableOp"
  input: "Mul_1984/ReadVariableOp/Enter"
  input: "add_1244"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1599"
  op: "ReadVariableOp"
  input: "Mul_1984/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1096"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1097"
  op: "AssignVariableOp"
  input: "Mul_1986/ReadVariableOp/Enter"
  input: "add_1245"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1600"
  op: "ReadVariableOp"
  input: "Mul_1986/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1097"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1989/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1989/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1989/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1989/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1989"
  op: "Mul"
  input: "Mul_1989/x"
  input: "Mul_1989/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1990/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1990"
  op: "Mul"
  input: "Mul_1990/x"
  input: "CrossReplicaSum_366"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1247"
  op: "AddV2"
  input: "Mul_1989"
  input: "Mul_1990"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1991/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1991/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1991/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1991/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1991"
  op: "Mul"
  input: "Mul_1991/x"
  input: "Mul_1991/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_366"
  op: "Square"
  input: "CrossReplicaSum_366"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1992/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1992"
  op: "Mul"
  input: "Mul_1992/x"
  input: "Square_366"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1248"
  op: "AddV2"
  input: "Mul_1991"
  input: "Mul_1992"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_366"
  op: "Sqrt"
  input: "add_1248"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1249/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1249"
  op: "AddV2"
  input: "Sqrt_366"
  input: "add_1249/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_369"
  op: "RealDiv"
  input: "add_1247"
  input: "add_1249"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1601"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_1993/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_1993"
  op: "Mul"
  input: "mul_1993/x"
  input: "ReadVariableOp_1601"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1250"
  op: "AddV2"
  input: "truediv_369"
  input: "mul_1993"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1994"
  op: "Mul"
  input: "mul_21"
  input: "add_1250"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1602"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_370"
  op: "Sub"
  input: "ReadVariableOp_1602"
  input: "mul_1994"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1098"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_370"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1603"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1098"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1099"
  op: "AssignVariableOp"
  input: "Mul_1989/ReadVariableOp/Enter"
  input: "add_1247"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1604"
  op: "ReadVariableOp"
  input: "Mul_1989/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1099"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1100"
  op: "AssignVariableOp"
  input: "Mul_1991/ReadVariableOp/Enter"
  input: "add_1248"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1605"
  op: "ReadVariableOp"
  input: "Mul_1991/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1100"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1995/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_1995/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1995/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1995/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1995"
  op: "Mul"
  input: "Mul_1995/x"
  input: "Mul_1995/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1996/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_1996"
  op: "Mul"
  input: "Mul_1996/x"
  input: "CrossReplicaSum_367"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1251"
  op: "AddV2"
  input: "Mul_1995"
  input: "Mul_1996"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1997/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_1997/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_1997/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_1997/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_1997"
  op: "Mul"
  input: "Mul_1997/x"
  input: "Mul_1997/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_367"
  op: "Square"
  input: "CrossReplicaSum_367"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_1998/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_1998"
  op: "Mul"
  input: "Mul_1998/x"
  input: "Square_367"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1252"
  op: "AddV2"
  input: "Mul_1997"
  input: "Mul_1998"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_367"
  op: "Sqrt"
  input: "add_1252"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1253/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1253"
  op: "AddV2"
  input: "Sqrt_367"
  input: "add_1253/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_370"
  op: "RealDiv"
  input: "add_1251"
  input: "add_1253"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_1999"
  op: "Mul"
  input: "mul_21"
  input: "truediv_370"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1606"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_371"
  op: "Sub"
  input: "ReadVariableOp_1606"
  input: "mul_1999"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1101"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_371"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1607"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1101"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1102"
  op: "AssignVariableOp"
  input: "Mul_1995/ReadVariableOp/Enter"
  input: "add_1251"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1608"
  op: "ReadVariableOp"
  input: "Mul_1995/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1102"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1103"
  op: "AssignVariableOp"
  input: "Mul_1997/ReadVariableOp/Enter"
  input: "add_1252"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1609"
  op: "ReadVariableOp"
  input: "Mul_1997/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1103"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2000/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2000/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2000/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2000/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2000"
  op: "Mul"
  input: "Mul_2000/x"
  input: "Mul_2000/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2001/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2001"
  op: "Mul"
  input: "Mul_2001/x"
  input: "CrossReplicaSum_368"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1254"
  op: "AddV2"
  input: "Mul_2000"
  input: "Mul_2001"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2002/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2002/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2002/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2002/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2002"
  op: "Mul"
  input: "Mul_2002/x"
  input: "Mul_2002/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_368"
  op: "Square"
  input: "CrossReplicaSum_368"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2003/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2003"
  op: "Mul"
  input: "Mul_2003/x"
  input: "Square_368"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1255"
  op: "AddV2"
  input: "Mul_2002"
  input: "Mul_2003"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_368"
  op: "Sqrt"
  input: "add_1255"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1256/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1256"
  op: "AddV2"
  input: "Sqrt_368"
  input: "add_1256/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_371"
  op: "RealDiv"
  input: "add_1254"
  input: "add_1256"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1610"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2004/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2004"
  op: "Mul"
  input: "mul_2004/x"
  input: "ReadVariableOp_1610"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1257"
  op: "AddV2"
  input: "truediv_371"
  input: "mul_2004"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2005"
  op: "Mul"
  input: "mul_21"
  input: "add_1257"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1611"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_372"
  op: "Sub"
  input: "ReadVariableOp_1611"
  input: "mul_2005"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1104"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_372"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1612"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1104"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1105"
  op: "AssignVariableOp"
  input: "Mul_2000/ReadVariableOp/Enter"
  input: "add_1254"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1613"
  op: "ReadVariableOp"
  input: "Mul_2000/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1105"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1106"
  op: "AssignVariableOp"
  input: "Mul_2002/ReadVariableOp/Enter"
  input: "add_1255"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1614"
  op: "ReadVariableOp"
  input: "Mul_2002/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1106"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2006/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2006/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2006/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2006/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2006"
  op: "Mul"
  input: "Mul_2006/x"
  input: "Mul_2006/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2007/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2007"
  op: "Mul"
  input: "Mul_2007/x"
  input: "CrossReplicaSum_369"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1258"
  op: "AddV2"
  input: "Mul_2006"
  input: "Mul_2007"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2008/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2008/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2008/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2008/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2008"
  op: "Mul"
  input: "Mul_2008/x"
  input: "Mul_2008/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_369"
  op: "Square"
  input: "CrossReplicaSum_369"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2009/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2009"
  op: "Mul"
  input: "Mul_2009/x"
  input: "Square_369"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1259"
  op: "AddV2"
  input: "Mul_2008"
  input: "Mul_2009"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_369"
  op: "Sqrt"
  input: "add_1259"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1260/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1260"
  op: "AddV2"
  input: "Sqrt_369"
  input: "add_1260/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_372"
  op: "RealDiv"
  input: "add_1258"
  input: "add_1260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2010"
  op: "Mul"
  input: "mul_21"
  input: "truediv_372"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1615"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_373"
  op: "Sub"
  input: "ReadVariableOp_1615"
  input: "mul_2010"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1107"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_373"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1616"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1107"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1108"
  op: "AssignVariableOp"
  input: "Mul_2006/ReadVariableOp/Enter"
  input: "add_1258"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1617"
  op: "ReadVariableOp"
  input: "Mul_2006/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1108"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1109"
  op: "AssignVariableOp"
  input: "Mul_2008/ReadVariableOp/Enter"
  input: "add_1259"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1618"
  op: "ReadVariableOp"
  input: "Mul_2008/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1109"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2011/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2011/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2011/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2011/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2011"
  op: "Mul"
  input: "Mul_2011/x"
  input: "Mul_2011/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2012/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2012"
  op: "Mul"
  input: "Mul_2012/x"
  input: "CrossReplicaSum_370"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1261"
  op: "AddV2"
  input: "Mul_2011"
  input: "Mul_2012"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2013/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2013/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2013/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2013/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2013"
  op: "Mul"
  input: "Mul_2013/x"
  input: "Mul_2013/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_370"
  op: "Square"
  input: "CrossReplicaSum_370"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2014/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2014"
  op: "Mul"
  input: "Mul_2014/x"
  input: "Square_370"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1262"
  op: "AddV2"
  input: "Mul_2013"
  input: "Mul_2014"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_370"
  op: "Sqrt"
  input: "add_1262"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1263/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1263"
  op: "AddV2"
  input: "Sqrt_370"
  input: "add_1263/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_373"
  op: "RealDiv"
  input: "add_1261"
  input: "add_1263"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1619"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2015/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2015"
  op: "Mul"
  input: "mul_2015/x"
  input: "ReadVariableOp_1619"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1264"
  op: "AddV2"
  input: "truediv_373"
  input: "mul_2015"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2016"
  op: "Mul"
  input: "mul_21"
  input: "add_1264"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1620"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_374"
  op: "Sub"
  input: "ReadVariableOp_1620"
  input: "mul_2016"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1110"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_374"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1621"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1110"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1111"
  op: "AssignVariableOp"
  input: "Mul_2011/ReadVariableOp/Enter"
  input: "add_1261"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1622"
  op: "ReadVariableOp"
  input: "Mul_2011/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1111"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1112"
  op: "AssignVariableOp"
  input: "Mul_2013/ReadVariableOp/Enter"
  input: "add_1262"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1623"
  op: "ReadVariableOp"
  input: "Mul_2013/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1112"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2017/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2017/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2017/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2017/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2017"
  op: "Mul"
  input: "Mul_2017/x"
  input: "Mul_2017/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2018/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2018"
  op: "Mul"
  input: "Mul_2018/x"
  input: "CrossReplicaSum_371"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1265"
  op: "AddV2"
  input: "Mul_2017"
  input: "Mul_2018"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2019/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2019/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2019/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2019/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2019"
  op: "Mul"
  input: "Mul_2019/x"
  input: "Mul_2019/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_371"
  op: "Square"
  input: "CrossReplicaSum_371"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2020/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2020"
  op: "Mul"
  input: "Mul_2020/x"
  input: "Square_371"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1266"
  op: "AddV2"
  input: "Mul_2019"
  input: "Mul_2020"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_371"
  op: "Sqrt"
  input: "add_1266"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1267/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1267"
  op: "AddV2"
  input: "Sqrt_371"
  input: "add_1267/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_374"
  op: "RealDiv"
  input: "add_1265"
  input: "add_1267"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2021"
  op: "Mul"
  input: "mul_21"
  input: "truediv_374"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1624"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_375"
  op: "Sub"
  input: "ReadVariableOp_1624"
  input: "mul_2021"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1113"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_375"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1625"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1113"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1114"
  op: "AssignVariableOp"
  input: "Mul_2017/ReadVariableOp/Enter"
  input: "add_1265"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1626"
  op: "ReadVariableOp"
  input: "Mul_2017/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1114"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1115"
  op: "AssignVariableOp"
  input: "Mul_2019/ReadVariableOp/Enter"
  input: "add_1266"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1627"
  op: "ReadVariableOp"
  input: "Mul_2019/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1115"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2022/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2022/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2022/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2022/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2022"
  op: "Mul"
  input: "Mul_2022/x"
  input: "Mul_2022/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2023/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2023"
  op: "Mul"
  input: "Mul_2023/x"
  input: "CrossReplicaSum_372"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1268"
  op: "AddV2"
  input: "Mul_2022"
  input: "Mul_2023"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2024/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2024/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2024/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2024/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2024"
  op: "Mul"
  input: "Mul_2024/x"
  input: "Mul_2024/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_372"
  op: "Square"
  input: "CrossReplicaSum_372"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2025/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2025"
  op: "Mul"
  input: "Mul_2025/x"
  input: "Square_372"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1269"
  op: "AddV2"
  input: "Mul_2024"
  input: "Mul_2025"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_372"
  op: "Sqrt"
  input: "add_1269"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1270/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1270"
  op: "AddV2"
  input: "Sqrt_372"
  input: "add_1270/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_375"
  op: "RealDiv"
  input: "add_1268"
  input: "add_1270"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1628"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2026/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2026"
  op: "Mul"
  input: "mul_2026/x"
  input: "ReadVariableOp_1628"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1271"
  op: "AddV2"
  input: "truediv_375"
  input: "mul_2026"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2027"
  op: "Mul"
  input: "mul_21"
  input: "add_1271"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1629"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_376"
  op: "Sub"
  input: "ReadVariableOp_1629"
  input: "mul_2027"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1116"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_376"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1630"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1116"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1117"
  op: "AssignVariableOp"
  input: "Mul_2022/ReadVariableOp/Enter"
  input: "add_1268"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1631"
  op: "ReadVariableOp"
  input: "Mul_2022/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1117"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1118"
  op: "AssignVariableOp"
  input: "Mul_2024/ReadVariableOp/Enter"
  input: "add_1269"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1632"
  op: "ReadVariableOp"
  input: "Mul_2024/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1118"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2028/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2028/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2028/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2028/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2028"
  op: "Mul"
  input: "Mul_2028/x"
  input: "Mul_2028/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2029/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2029"
  op: "Mul"
  input: "Mul_2029/x"
  input: "CrossReplicaSum_373"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1272"
  op: "AddV2"
  input: "Mul_2028"
  input: "Mul_2029"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2030/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2030/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2030/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2030/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2030"
  op: "Mul"
  input: "Mul_2030/x"
  input: "Mul_2030/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_373"
  op: "Square"
  input: "CrossReplicaSum_373"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2031/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2031"
  op: "Mul"
  input: "Mul_2031/x"
  input: "Square_373"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1273"
  op: "AddV2"
  input: "Mul_2030"
  input: "Mul_2031"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_373"
  op: "Sqrt"
  input: "add_1273"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1274/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1274"
  op: "AddV2"
  input: "Sqrt_373"
  input: "add_1274/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_376"
  op: "RealDiv"
  input: "add_1272"
  input: "add_1274"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2032"
  op: "Mul"
  input: "mul_21"
  input: "truediv_376"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1633"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_377"
  op: "Sub"
  input: "ReadVariableOp_1633"
  input: "mul_2032"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1119"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_377"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1634"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1119"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1120"
  op: "AssignVariableOp"
  input: "Mul_2028/ReadVariableOp/Enter"
  input: "add_1272"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1635"
  op: "ReadVariableOp"
  input: "Mul_2028/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1120"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1121"
  op: "AssignVariableOp"
  input: "Mul_2030/ReadVariableOp/Enter"
  input: "add_1273"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1636"
  op: "ReadVariableOp"
  input: "Mul_2030/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1121"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2033/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2033/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2033/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2033/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2033"
  op: "Mul"
  input: "Mul_2033/x"
  input: "Mul_2033/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2034/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2034"
  op: "Mul"
  input: "Mul_2034/x"
  input: "CrossReplicaSum_374"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1275"
  op: "AddV2"
  input: "Mul_2033"
  input: "Mul_2034"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2035/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2035/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2035/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2035/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2035"
  op: "Mul"
  input: "Mul_2035/x"
  input: "Mul_2035/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_374"
  op: "Square"
  input: "CrossReplicaSum_374"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2036/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2036"
  op: "Mul"
  input: "Mul_2036/x"
  input: "Square_374"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1276"
  op: "AddV2"
  input: "Mul_2035"
  input: "Mul_2036"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_374"
  op: "Sqrt"
  input: "add_1276"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1277/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1277"
  op: "AddV2"
  input: "Sqrt_374"
  input: "add_1277/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_377"
  op: "RealDiv"
  input: "add_1275"
  input: "add_1277"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2037"
  op: "Mul"
  input: "mul_21"
  input: "truediv_377"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1637"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_378"
  op: "Sub"
  input: "ReadVariableOp_1637"
  input: "mul_2037"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1122"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_378"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1638"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1122"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1123"
  op: "AssignVariableOp"
  input: "Mul_2033/ReadVariableOp/Enter"
  input: "add_1275"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1639"
  op: "ReadVariableOp"
  input: "Mul_2033/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1123"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1124"
  op: "AssignVariableOp"
  input: "Mul_2035/ReadVariableOp/Enter"
  input: "add_1276"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1640"
  op: "ReadVariableOp"
  input: "Mul_2035/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1124"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2038/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2038/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2038/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2038/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2038"
  op: "Mul"
  input: "Mul_2038/x"
  input: "Mul_2038/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2039/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2039"
  op: "Mul"
  input: "Mul_2039/x"
  input: "CrossReplicaSum_375"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1278"
  op: "AddV2"
  input: "Mul_2038"
  input: "Mul_2039"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2040/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2040/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2040/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2040/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2040"
  op: "Mul"
  input: "Mul_2040/x"
  input: "Mul_2040/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_375"
  op: "Square"
  input: "CrossReplicaSum_375"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2041/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2041"
  op: "Mul"
  input: "Mul_2041/x"
  input: "Square_375"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1279"
  op: "AddV2"
  input: "Mul_2040"
  input: "Mul_2041"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_375"
  op: "Sqrt"
  input: "add_1279"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1280/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1280"
  op: "AddV2"
  input: "Sqrt_375"
  input: "add_1280/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_378"
  op: "RealDiv"
  input: "add_1278"
  input: "add_1280"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2042"
  op: "Mul"
  input: "mul_21"
  input: "truediv_378"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1641"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_379"
  op: "Sub"
  input: "ReadVariableOp_1641"
  input: "mul_2042"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1125"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_379"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1642"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1125"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1126"
  op: "AssignVariableOp"
  input: "Mul_2038/ReadVariableOp/Enter"
  input: "add_1278"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1643"
  op: "ReadVariableOp"
  input: "Mul_2038/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1126"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1127"
  op: "AssignVariableOp"
  input: "Mul_2040/ReadVariableOp/Enter"
  input: "add_1279"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1644"
  op: "ReadVariableOp"
  input: "Mul_2040/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1127"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2043/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2043/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2043/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2043/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2043"
  op: "Mul"
  input: "Mul_2043/x"
  input: "Mul_2043/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2044/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2044"
  op: "Mul"
  input: "Mul_2044/x"
  input: "CrossReplicaSum_376"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1281"
  op: "AddV2"
  input: "Mul_2043"
  input: "Mul_2044"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2045/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2045/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2045/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2045/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2045"
  op: "Mul"
  input: "Mul_2045/x"
  input: "Mul_2045/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_376"
  op: "Square"
  input: "CrossReplicaSum_376"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2046/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2046"
  op: "Mul"
  input: "Mul_2046/x"
  input: "Square_376"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1282"
  op: "AddV2"
  input: "Mul_2045"
  input: "Mul_2046"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_376"
  op: "Sqrt"
  input: "add_1282"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1283/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1283"
  op: "AddV2"
  input: "Sqrt_376"
  input: "add_1283/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_379"
  op: "RealDiv"
  input: "add_1281"
  input: "add_1283"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1645"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2047/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2047"
  op: "Mul"
  input: "mul_2047/x"
  input: "ReadVariableOp_1645"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1284"
  op: "AddV2"
  input: "truediv_379"
  input: "mul_2047"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2048"
  op: "Mul"
  input: "mul_21"
  input: "add_1284"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1646"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_380"
  op: "Sub"
  input: "ReadVariableOp_1646"
  input: "mul_2048"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1128"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_380"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1647"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1128"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1129"
  op: "AssignVariableOp"
  input: "Mul_2043/ReadVariableOp/Enter"
  input: "add_1281"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1648"
  op: "ReadVariableOp"
  input: "Mul_2043/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1129"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1130"
  op: "AssignVariableOp"
  input: "Mul_2045/ReadVariableOp/Enter"
  input: "add_1282"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1649"
  op: "ReadVariableOp"
  input: "Mul_2045/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1130"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2049/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2049/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2049/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2049/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2049"
  op: "Mul"
  input: "Mul_2049/x"
  input: "Mul_2049/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2050/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2050"
  op: "Mul"
  input: "Mul_2050/x"
  input: "CrossReplicaSum_377"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1285"
  op: "AddV2"
  input: "Mul_2049"
  input: "Mul_2050"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2051/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2051/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2051/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2051/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2051"
  op: "Mul"
  input: "Mul_2051/x"
  input: "Mul_2051/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_377"
  op: "Square"
  input: "CrossReplicaSum_377"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2052/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2052"
  op: "Mul"
  input: "Mul_2052/x"
  input: "Square_377"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1286"
  op: "AddV2"
  input: "Mul_2051"
  input: "Mul_2052"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_377"
  op: "Sqrt"
  input: "add_1286"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1287/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1287"
  op: "AddV2"
  input: "Sqrt_377"
  input: "add_1287/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_380"
  op: "RealDiv"
  input: "add_1285"
  input: "add_1287"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2053"
  op: "Mul"
  input: "mul_21"
  input: "truediv_380"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1650"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_381"
  op: "Sub"
  input: "ReadVariableOp_1650"
  input: "mul_2053"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1131"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_381"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1651"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1131"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1132"
  op: "AssignVariableOp"
  input: "Mul_2049/ReadVariableOp/Enter"
  input: "add_1285"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1652"
  op: "ReadVariableOp"
  input: "Mul_2049/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1132"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1133"
  op: "AssignVariableOp"
  input: "Mul_2051/ReadVariableOp/Enter"
  input: "add_1286"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1653"
  op: "ReadVariableOp"
  input: "Mul_2051/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1133"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2054/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2054/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2054/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2054/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2054"
  op: "Mul"
  input: "Mul_2054/x"
  input: "Mul_2054/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2055/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2055"
  op: "Mul"
  input: "Mul_2055/x"
  input: "CrossReplicaSum_378"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1288"
  op: "AddV2"
  input: "Mul_2054"
  input: "Mul_2055"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2056/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2056/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2056/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2056/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2056"
  op: "Mul"
  input: "Mul_2056/x"
  input: "Mul_2056/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_378"
  op: "Square"
  input: "CrossReplicaSum_378"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2057/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2057"
  op: "Mul"
  input: "Mul_2057/x"
  input: "Square_378"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1289"
  op: "AddV2"
  input: "Mul_2056"
  input: "Mul_2057"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_378"
  op: "Sqrt"
  input: "add_1289"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1290/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1290"
  op: "AddV2"
  input: "Sqrt_378"
  input: "add_1290/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_381"
  op: "RealDiv"
  input: "add_1288"
  input: "add_1290"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1654"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2058/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2058"
  op: "Mul"
  input: "mul_2058/x"
  input: "ReadVariableOp_1654"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1291"
  op: "AddV2"
  input: "truediv_381"
  input: "mul_2058"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2059"
  op: "Mul"
  input: "mul_21"
  input: "add_1291"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1655"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_382"
  op: "Sub"
  input: "ReadVariableOp_1655"
  input: "mul_2059"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1134"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_382"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1656"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1134"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1135"
  op: "AssignVariableOp"
  input: "Mul_2054/ReadVariableOp/Enter"
  input: "add_1288"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1657"
  op: "ReadVariableOp"
  input: "Mul_2054/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1135"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1136"
  op: "AssignVariableOp"
  input: "Mul_2056/ReadVariableOp/Enter"
  input: "add_1289"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1658"
  op: "ReadVariableOp"
  input: "Mul_2056/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1136"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_m"
  input: "electra/encoder/layer_10/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_v"
  input: "electra/encoder/layer_10/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2060/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2060/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2060/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2060/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2060"
  op: "Mul"
  input: "Mul_2060/x"
  input: "Mul_2060/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2061/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2061"
  op: "Mul"
  input: "Mul_2061/x"
  input: "CrossReplicaSum_379"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1292"
  op: "AddV2"
  input: "Mul_2060"
  input: "Mul_2061"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2062/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2062/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2062/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2062/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2062"
  op: "Mul"
  input: "Mul_2062/x"
  input: "Mul_2062/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_379"
  op: "Square"
  input: "CrossReplicaSum_379"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2063/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2063"
  op: "Mul"
  input: "Mul_2063/x"
  input: "Square_379"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1293"
  op: "AddV2"
  input: "Mul_2062"
  input: "Mul_2063"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_379"
  op: "Sqrt"
  input: "add_1293"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1294/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1294"
  op: "AddV2"
  input: "Sqrt_379"
  input: "add_1294/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_382"
  op: "RealDiv"
  input: "add_1292"
  input: "add_1294"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2064"
  op: "Mul"
  input: "mul_21"
  input: "truediv_382"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1659"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_383"
  op: "Sub"
  input: "ReadVariableOp_1659"
  input: "mul_2064"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1137"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_383"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1660"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1137"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1138"
  op: "AssignVariableOp"
  input: "Mul_2060/ReadVariableOp/Enter"
  input: "add_1292"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1661"
  op: "ReadVariableOp"
  input: "Mul_2060/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1138"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1139"
  op: "AssignVariableOp"
  input: "Mul_2062/ReadVariableOp/Enter"
  input: "add_1293"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1662"
  op: "ReadVariableOp"
  input: "Mul_2062/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1139"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2065/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2065/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2065/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2065/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2065"
  op: "Mul"
  input: "Mul_2065/x"
  input: "Mul_2065/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2066/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2066"
  op: "Mul"
  input: "Mul_2066/x"
  input: "CrossReplicaSum_380"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1295"
  op: "AddV2"
  input: "Mul_2065"
  input: "Mul_2066"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2067/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2067/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2067/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2067/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2067"
  op: "Mul"
  input: "Mul_2067/x"
  input: "Mul_2067/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_380"
  op: "Square"
  input: "CrossReplicaSum_380"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2068/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2068"
  op: "Mul"
  input: "Mul_2068/x"
  input: "Square_380"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1296"
  op: "AddV2"
  input: "Mul_2067"
  input: "Mul_2068"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_380"
  op: "Sqrt"
  input: "add_1296"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1297/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1297"
  op: "AddV2"
  input: "Sqrt_380"
  input: "add_1297/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_383"
  op: "RealDiv"
  input: "add_1295"
  input: "add_1297"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2069"
  op: "Mul"
  input: "mul_21"
  input: "truediv_383"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1663"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_384"
  op: "Sub"
  input: "ReadVariableOp_1663"
  input: "mul_2069"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1140"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_384"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1664"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1140"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1141"
  op: "AssignVariableOp"
  input: "Mul_2065/ReadVariableOp/Enter"
  input: "add_1295"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1665"
  op: "ReadVariableOp"
  input: "Mul_2065/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1141"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1142"
  op: "AssignVariableOp"
  input: "Mul_2067/ReadVariableOp/Enter"
  input: "add_1296"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1666"
  op: "ReadVariableOp"
  input: "Mul_2067/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1142"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2070/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2070/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2070/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2070/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2070"
  op: "Mul"
  input: "Mul_2070/x"
  input: "Mul_2070/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2071/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2071"
  op: "Mul"
  input: "Mul_2071/x"
  input: "CrossReplicaSum_381"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1298"
  op: "AddV2"
  input: "Mul_2070"
  input: "Mul_2071"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2072/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2072/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2072/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2072/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2072"
  op: "Mul"
  input: "Mul_2072/x"
  input: "Mul_2072/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_381"
  op: "Square"
  input: "CrossReplicaSum_381"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2073/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2073"
  op: "Mul"
  input: "Mul_2073/x"
  input: "Square_381"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1299"
  op: "AddV2"
  input: "Mul_2072"
  input: "Mul_2073"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_381"
  op: "Sqrt"
  input: "add_1299"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1300/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1300"
  op: "AddV2"
  input: "Sqrt_381"
  input: "add_1300/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_384"
  op: "RealDiv"
  input: "add_1298"
  input: "add_1300"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2074"
  op: "Mul"
  input: "mul_21"
  input: "truediv_384"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1667"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_385"
  op: "Sub"
  input: "ReadVariableOp_1667"
  input: "mul_2074"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1143"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_385"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1668"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1143"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1144"
  op: "AssignVariableOp"
  input: "Mul_2070/ReadVariableOp/Enter"
  input: "add_1298"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1669"
  op: "ReadVariableOp"
  input: "Mul_2070/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1144"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1145"
  op: "AssignVariableOp"
  input: "Mul_2072/ReadVariableOp/Enter"
  input: "add_1299"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1670"
  op: "ReadVariableOp"
  input: "Mul_2072/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1145"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2075/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2075/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2075/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2075/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2075"
  op: "Mul"
  input: "Mul_2075/x"
  input: "Mul_2075/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2076/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2076"
  op: "Mul"
  input: "Mul_2076/x"
  input: "CrossReplicaSum_382"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1301"
  op: "AddV2"
  input: "Mul_2075"
  input: "Mul_2076"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2077/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2077/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2077/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2077/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2077"
  op: "Mul"
  input: "Mul_2077/x"
  input: "Mul_2077/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_382"
  op: "Square"
  input: "CrossReplicaSum_382"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2078/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2078"
  op: "Mul"
  input: "Mul_2078/x"
  input: "Square_382"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1302"
  op: "AddV2"
  input: "Mul_2077"
  input: "Mul_2078"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_382"
  op: "Sqrt"
  input: "add_1302"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1303/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1303"
  op: "AddV2"
  input: "Sqrt_382"
  input: "add_1303/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_385"
  op: "RealDiv"
  input: "add_1301"
  input: "add_1303"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1671"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2079/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2079"
  op: "Mul"
  input: "mul_2079/x"
  input: "ReadVariableOp_1671"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1304"
  op: "AddV2"
  input: "truediv_385"
  input: "mul_2079"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2080"
  op: "Mul"
  input: "mul_21"
  input: "add_1304"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1672"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_386"
  op: "Sub"
  input: "ReadVariableOp_1672"
  input: "mul_2080"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1146"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "sub_386"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1673"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1146"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1147"
  op: "AssignVariableOp"
  input: "Mul_2075/ReadVariableOp/Enter"
  input: "add_1301"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1674"
  op: "ReadVariableOp"
  input: "Mul_2075/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1147"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1148"
  op: "AssignVariableOp"
  input: "Mul_2077/ReadVariableOp/Enter"
  input: "add_1302"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1675"
  op: "ReadVariableOp"
  input: "Mul_2077/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1148"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/query/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2081/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2081/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2081/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2081/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2081"
  op: "Mul"
  input: "Mul_2081/x"
  input: "Mul_2081/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2082/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2082"
  op: "Mul"
  input: "Mul_2082/x"
  input: "CrossReplicaSum_383"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1305"
  op: "AddV2"
  input: "Mul_2081"
  input: "Mul_2082"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2083/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2083/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2083/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2083/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2083"
  op: "Mul"
  input: "Mul_2083/x"
  input: "Mul_2083/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_383"
  op: "Square"
  input: "CrossReplicaSum_383"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2084/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2084"
  op: "Mul"
  input: "Mul_2084/x"
  input: "Square_383"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1306"
  op: "AddV2"
  input: "Mul_2083"
  input: "Mul_2084"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_383"
  op: "Sqrt"
  input: "add_1306"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1307/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1307"
  op: "AddV2"
  input: "Sqrt_383"
  input: "add_1307/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_386"
  op: "RealDiv"
  input: "add_1305"
  input: "add_1307"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2085"
  op: "Mul"
  input: "mul_21"
  input: "truediv_386"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1676"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_387"
  op: "Sub"
  input: "ReadVariableOp_1676"
  input: "mul_2085"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1149"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "sub_387"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1677"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1149"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1150"
  op: "AssignVariableOp"
  input: "Mul_2081/ReadVariableOp/Enter"
  input: "add_1305"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1678"
  op: "ReadVariableOp"
  input: "Mul_2081/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1150"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1151"
  op: "AssignVariableOp"
  input: "Mul_2083/ReadVariableOp/Enter"
  input: "add_1306"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1679"
  op: "ReadVariableOp"
  input: "Mul_2083/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1151"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2086/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2086/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2086/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2086/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2086"
  op: "Mul"
  input: "Mul_2086/x"
  input: "Mul_2086/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2087/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2087"
  op: "Mul"
  input: "Mul_2087/x"
  input: "CrossReplicaSum_384"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1308"
  op: "AddV2"
  input: "Mul_2086"
  input: "Mul_2087"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2088/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2088/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2088/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2088/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2088"
  op: "Mul"
  input: "Mul_2088/x"
  input: "Mul_2088/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_384"
  op: "Square"
  input: "CrossReplicaSum_384"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2089/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2089"
  op: "Mul"
  input: "Mul_2089/x"
  input: "Square_384"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1309"
  op: "AddV2"
  input: "Mul_2088"
  input: "Mul_2089"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_384"
  op: "Sqrt"
  input: "add_1309"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1310/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1310"
  op: "AddV2"
  input: "Sqrt_384"
  input: "add_1310/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_387"
  op: "RealDiv"
  input: "add_1308"
  input: "add_1310"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1680"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2090/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2090"
  op: "Mul"
  input: "mul_2090/x"
  input: "ReadVariableOp_1680"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1311"
  op: "AddV2"
  input: "truediv_387"
  input: "mul_2090"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2091"
  op: "Mul"
  input: "mul_21"
  input: "add_1311"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1681"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_388"
  op: "Sub"
  input: "ReadVariableOp_1681"
  input: "mul_2091"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1152"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "sub_388"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1682"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1152"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1153"
  op: "AssignVariableOp"
  input: "Mul_2086/ReadVariableOp/Enter"
  input: "add_1308"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1683"
  op: "ReadVariableOp"
  input: "Mul_2086/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1153"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1154"
  op: "AssignVariableOp"
  input: "Mul_2088/ReadVariableOp/Enter"
  input: "add_1309"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1684"
  op: "ReadVariableOp"
  input: "Mul_2088/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1154"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/key/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2092/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2092/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2092/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2092/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2092"
  op: "Mul"
  input: "Mul_2092/x"
  input: "Mul_2092/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2093/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2093"
  op: "Mul"
  input: "Mul_2093/x"
  input: "CrossReplicaSum_385"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1312"
  op: "AddV2"
  input: "Mul_2092"
  input: "Mul_2093"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2094/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2094/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2094/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2094/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2094"
  op: "Mul"
  input: "Mul_2094/x"
  input: "Mul_2094/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_385"
  op: "Square"
  input: "CrossReplicaSum_385"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2095/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2095"
  op: "Mul"
  input: "Mul_2095/x"
  input: "Square_385"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1313"
  op: "AddV2"
  input: "Mul_2094"
  input: "Mul_2095"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_385"
  op: "Sqrt"
  input: "add_1313"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1314/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1314"
  op: "AddV2"
  input: "Sqrt_385"
  input: "add_1314/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_388"
  op: "RealDiv"
  input: "add_1312"
  input: "add_1314"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2096"
  op: "Mul"
  input: "mul_21"
  input: "truediv_388"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1685"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_389"
  op: "Sub"
  input: "ReadVariableOp_1685"
  input: "mul_2096"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1155"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "sub_389"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1686"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1155"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1156"
  op: "AssignVariableOp"
  input: "Mul_2092/ReadVariableOp/Enter"
  input: "add_1312"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1687"
  op: "ReadVariableOp"
  input: "Mul_2092/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1156"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1157"
  op: "AssignVariableOp"
  input: "Mul_2094/ReadVariableOp/Enter"
  input: "add_1313"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1688"
  op: "ReadVariableOp"
  input: "Mul_2094/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1157"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2097/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2097/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2097/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2097/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2097"
  op: "Mul"
  input: "Mul_2097/x"
  input: "Mul_2097/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2098/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2098"
  op: "Mul"
  input: "Mul_2098/x"
  input: "CrossReplicaSum_386"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1315"
  op: "AddV2"
  input: "Mul_2097"
  input: "Mul_2098"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2099/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2099/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2099/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2099/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2099"
  op: "Mul"
  input: "Mul_2099/x"
  input: "Mul_2099/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_386"
  op: "Square"
  input: "CrossReplicaSum_386"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2100/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2100"
  op: "Mul"
  input: "Mul_2100/x"
  input: "Square_386"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1316"
  op: "AddV2"
  input: "Mul_2099"
  input: "Mul_2100"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_386"
  op: "Sqrt"
  input: "add_1316"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1317/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1317"
  op: "AddV2"
  input: "Sqrt_386"
  input: "add_1317/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_389"
  op: "RealDiv"
  input: "add_1315"
  input: "add_1317"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1689"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2101/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2101"
  op: "Mul"
  input: "mul_2101/x"
  input: "ReadVariableOp_1689"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1318"
  op: "AddV2"
  input: "truediv_389"
  input: "mul_2101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2102"
  op: "Mul"
  input: "mul_21"
  input: "add_1318"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1690"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_390"
  op: "Sub"
  input: "ReadVariableOp_1690"
  input: "mul_2102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1158"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "sub_390"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1691"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1158"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1159"
  op: "AssignVariableOp"
  input: "Mul_2097/ReadVariableOp/Enter"
  input: "add_1315"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1692"
  op: "ReadVariableOp"
  input: "Mul_2097/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1159"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1160"
  op: "AssignVariableOp"
  input: "Mul_2099/ReadVariableOp/Enter"
  input: "add_1316"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1693"
  op: "ReadVariableOp"
  input: "Mul_2099/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1160"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/self/value/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2103/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2103/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2103/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2103/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2103"
  op: "Mul"
  input: "Mul_2103/x"
  input: "Mul_2103/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2104/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2104"
  op: "Mul"
  input: "Mul_2104/x"
  input: "CrossReplicaSum_387"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1319"
  op: "AddV2"
  input: "Mul_2103"
  input: "Mul_2104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2105/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2105/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2105/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2105/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2105"
  op: "Mul"
  input: "Mul_2105/x"
  input: "Mul_2105/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_387"
  op: "Square"
  input: "CrossReplicaSum_387"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2106/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2106"
  op: "Mul"
  input: "Mul_2106/x"
  input: "Square_387"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1320"
  op: "AddV2"
  input: "Mul_2105"
  input: "Mul_2106"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_387"
  op: "Sqrt"
  input: "add_1320"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1321/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1321"
  op: "AddV2"
  input: "Sqrt_387"
  input: "add_1321/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_390"
  op: "RealDiv"
  input: "add_1319"
  input: "add_1321"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2107"
  op: "Mul"
  input: "mul_21"
  input: "truediv_390"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1694"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_391"
  op: "Sub"
  input: "ReadVariableOp_1694"
  input: "mul_2107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1161"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "sub_391"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1695"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1161"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1162"
  op: "AssignVariableOp"
  input: "Mul_2103/ReadVariableOp/Enter"
  input: "add_1319"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1696"
  op: "ReadVariableOp"
  input: "Mul_2103/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1162"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1163"
  op: "AssignVariableOp"
  input: "Mul_2105/ReadVariableOp/Enter"
  input: "add_1320"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1697"
  op: "ReadVariableOp"
  input: "Mul_2105/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1163"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2108/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2108/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2108/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2108/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2108"
  op: "Mul"
  input: "Mul_2108/x"
  input: "Mul_2108/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2109/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2109"
  op: "Mul"
  input: "Mul_2109/x"
  input: "CrossReplicaSum_388"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1322"
  op: "AddV2"
  input: "Mul_2108"
  input: "Mul_2109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2110/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2110/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2110/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2110/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2110"
  op: "Mul"
  input: "Mul_2110/x"
  input: "Mul_2110/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_388"
  op: "Square"
  input: "CrossReplicaSum_388"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2111/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2111"
  op: "Mul"
  input: "Mul_2111/x"
  input: "Square_388"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1323"
  op: "AddV2"
  input: "Mul_2110"
  input: "Mul_2111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_388"
  op: "Sqrt"
  input: "add_1323"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1324/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1324"
  op: "AddV2"
  input: "Sqrt_388"
  input: "add_1324/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_391"
  op: "RealDiv"
  input: "add_1322"
  input: "add_1324"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1698"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2112/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2112"
  op: "Mul"
  input: "mul_2112/x"
  input: "ReadVariableOp_1698"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1325"
  op: "AddV2"
  input: "truediv_391"
  input: "mul_2112"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2113"
  op: "Mul"
  input: "mul_21"
  input: "add_1325"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1699"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_392"
  op: "Sub"
  input: "ReadVariableOp_1699"
  input: "mul_2113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1164"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_392"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1700"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1164"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1165"
  op: "AssignVariableOp"
  input: "Mul_2108/ReadVariableOp/Enter"
  input: "add_1322"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1701"
  op: "ReadVariableOp"
  input: "Mul_2108/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1165"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1166"
  op: "AssignVariableOp"
  input: "Mul_2110/ReadVariableOp/Enter"
  input: "add_1323"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1702"
  op: "ReadVariableOp"
  input: "Mul_2110/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1166"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2114/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2114/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2114/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2114/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2114"
  op: "Mul"
  input: "Mul_2114/x"
  input: "Mul_2114/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2115/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2115"
  op: "Mul"
  input: "Mul_2115/x"
  input: "CrossReplicaSum_389"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1326"
  op: "AddV2"
  input: "Mul_2114"
  input: "Mul_2115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2116/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2116/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2116/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2116/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2116"
  op: "Mul"
  input: "Mul_2116/x"
  input: "Mul_2116/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_389"
  op: "Square"
  input: "CrossReplicaSum_389"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2117/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2117"
  op: "Mul"
  input: "Mul_2117/x"
  input: "Square_389"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1327"
  op: "AddV2"
  input: "Mul_2116"
  input: "Mul_2117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_389"
  op: "Sqrt"
  input: "add_1327"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1328/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1328"
  op: "AddV2"
  input: "Sqrt_389"
  input: "add_1328/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_392"
  op: "RealDiv"
  input: "add_1326"
  input: "add_1328"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2118"
  op: "Mul"
  input: "mul_21"
  input: "truediv_392"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1703"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_393"
  op: "Sub"
  input: "ReadVariableOp_1703"
  input: "mul_2118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1167"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_393"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1704"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1167"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1168"
  op: "AssignVariableOp"
  input: "Mul_2114/ReadVariableOp/Enter"
  input: "add_1326"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1705"
  op: "ReadVariableOp"
  input: "Mul_2114/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1168"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1169"
  op: "AssignVariableOp"
  input: "Mul_2116/ReadVariableOp/Enter"
  input: "add_1327"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1706"
  op: "ReadVariableOp"
  input: "Mul_2116/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1169"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2119/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2119/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2119/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2119/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2119"
  op: "Mul"
  input: "Mul_2119/x"
  input: "Mul_2119/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2120/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2120"
  op: "Mul"
  input: "Mul_2120/x"
  input: "CrossReplicaSum_390"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1329"
  op: "AddV2"
  input: "Mul_2119"
  input: "Mul_2120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2121/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2121/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2121/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2121/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2121"
  op: "Mul"
  input: "Mul_2121/x"
  input: "Mul_2121/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_390"
  op: "Square"
  input: "CrossReplicaSum_390"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2122/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2122"
  op: "Mul"
  input: "Mul_2122/x"
  input: "Square_390"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1330"
  op: "AddV2"
  input: "Mul_2121"
  input: "Mul_2122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_390"
  op: "Sqrt"
  input: "add_1330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1331/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1331"
  op: "AddV2"
  input: "Sqrt_390"
  input: "add_1331/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_393"
  op: "RealDiv"
  input: "add_1329"
  input: "add_1331"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2123"
  op: "Mul"
  input: "mul_21"
  input: "truediv_393"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1707"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_394"
  op: "Sub"
  input: "ReadVariableOp_1707"
  input: "mul_2123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1170"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_394"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1708"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1170"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1171"
  op: "AssignVariableOp"
  input: "Mul_2119/ReadVariableOp/Enter"
  input: "add_1329"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1709"
  op: "ReadVariableOp"
  input: "Mul_2119/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1171"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1172"
  op: "AssignVariableOp"
  input: "Mul_2121/ReadVariableOp/Enter"
  input: "add_1330"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1710"
  op: "ReadVariableOp"
  input: "Mul_2121/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1172"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2124/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2124/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2124/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2124/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2124"
  op: "Mul"
  input: "Mul_2124/x"
  input: "Mul_2124/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2125/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2125"
  op: "Mul"
  input: "Mul_2125/x"
  input: "CrossReplicaSum_391"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1332"
  op: "AddV2"
  input: "Mul_2124"
  input: "Mul_2125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2126/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2126/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2126/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2126/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2126"
  op: "Mul"
  input: "Mul_2126/x"
  input: "Mul_2126/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_391"
  op: "Square"
  input: "CrossReplicaSum_391"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2127/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2127"
  op: "Mul"
  input: "Mul_2127/x"
  input: "Square_391"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1333"
  op: "AddV2"
  input: "Mul_2126"
  input: "Mul_2127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_391"
  op: "Sqrt"
  input: "add_1333"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1334/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1334"
  op: "AddV2"
  input: "Sqrt_391"
  input: "add_1334/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_394"
  op: "RealDiv"
  input: "add_1332"
  input: "add_1334"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2128"
  op: "Mul"
  input: "mul_21"
  input: "truediv_394"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1711"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_395"
  op: "Sub"
  input: "ReadVariableOp_1711"
  input: "mul_2128"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1173"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_395"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1712"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1173"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1174"
  op: "AssignVariableOp"
  input: "Mul_2124/ReadVariableOp/Enter"
  input: "add_1332"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1713"
  op: "ReadVariableOp"
  input: "Mul_2124/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1174"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1175"
  op: "AssignVariableOp"
  input: "Mul_2126/ReadVariableOp/Enter"
  input: "add_1333"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1714"
  op: "ReadVariableOp"
  input: "Mul_2126/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1175"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2129/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2129/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2129/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2129/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2129"
  op: "Mul"
  input: "Mul_2129/x"
  input: "Mul_2129/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2130/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2130"
  op: "Mul"
  input: "Mul_2130/x"
  input: "CrossReplicaSum_392"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1335"
  op: "AddV2"
  input: "Mul_2129"
  input: "Mul_2130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2131/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2131/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2131/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2131/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2131"
  op: "Mul"
  input: "Mul_2131/x"
  input: "Mul_2131/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_392"
  op: "Square"
  input: "CrossReplicaSum_392"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2132/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2132"
  op: "Mul"
  input: "Mul_2132/x"
  input: "Square_392"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1336"
  op: "AddV2"
  input: "Mul_2131"
  input: "Mul_2132"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_392"
  op: "Sqrt"
  input: "add_1336"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1337/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1337"
  op: "AddV2"
  input: "Sqrt_392"
  input: "add_1337/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_395"
  op: "RealDiv"
  input: "add_1335"
  input: "add_1337"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1715"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2133/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2133"
  op: "Mul"
  input: "mul_2133/x"
  input: "ReadVariableOp_1715"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1338"
  op: "AddV2"
  input: "truediv_395"
  input: "mul_2133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2134"
  op: "Mul"
  input: "mul_21"
  input: "add_1338"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1716"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_396"
  op: "Sub"
  input: "ReadVariableOp_1716"
  input: "mul_2134"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1176"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_396"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1717"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1176"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1177"
  op: "AssignVariableOp"
  input: "Mul_2129/ReadVariableOp/Enter"
  input: "add_1335"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1718"
  op: "ReadVariableOp"
  input: "Mul_2129/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1177"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1178"
  op: "AssignVariableOp"
  input: "Mul_2131/ReadVariableOp/Enter"
  input: "add_1336"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1719"
  op: "ReadVariableOp"
  input: "Mul_2131/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1178"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/intermediate/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2135/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2135/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2135/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2135/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2135"
  op: "Mul"
  input: "Mul_2135/x"
  input: "Mul_2135/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2136/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2136"
  op: "Mul"
  input: "Mul_2136/x"
  input: "CrossReplicaSum_393"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1339"
  op: "AddV2"
  input: "Mul_2135"
  input: "Mul_2136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2137/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2137/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2137/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2137/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2137"
  op: "Mul"
  input: "Mul_2137/x"
  input: "Mul_2137/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_393"
  op: "Square"
  input: "CrossReplicaSum_393"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2138/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2138"
  op: "Mul"
  input: "Mul_2138/x"
  input: "Square_393"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1340"
  op: "AddV2"
  input: "Mul_2137"
  input: "Mul_2138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_393"
  op: "Sqrt"
  input: "add_1340"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1341/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1341"
  op: "AddV2"
  input: "Sqrt_393"
  input: "add_1341/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_396"
  op: "RealDiv"
  input: "add_1339"
  input: "add_1341"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2139"
  op: "Mul"
  input: "mul_21"
  input: "truediv_396"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1720"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_397"
  op: "Sub"
  input: "ReadVariableOp_1720"
  input: "mul_2139"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1179"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_397"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1721"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1179"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1180"
  op: "AssignVariableOp"
  input: "Mul_2135/ReadVariableOp/Enter"
  input: "add_1339"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1722"
  op: "ReadVariableOp"
  input: "Mul_2135/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1180"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1181"
  op: "AssignVariableOp"
  input: "Mul_2137/ReadVariableOp/Enter"
  input: "add_1340"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1723"
  op: "ReadVariableOp"
  input: "Mul_2137/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1181"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/dense/kernel/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_m"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/dense/kernel/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_v"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2140/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2140/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2140/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2140/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2140"
  op: "Mul"
  input: "Mul_2140/x"
  input: "Mul_2140/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2141/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2141"
  op: "Mul"
  input: "Mul_2141/x"
  input: "CrossReplicaSum_394"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1342"
  op: "AddV2"
  input: "Mul_2140"
  input: "Mul_2141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2142/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2142/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2142/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2142/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2142"
  op: "Mul"
  input: "Mul_2142/x"
  input: "Mul_2142/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_394"
  op: "Square"
  input: "CrossReplicaSum_394"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2143/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2143"
  op: "Mul"
  input: "Mul_2143/x"
  input: "Square_394"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1343"
  op: "AddV2"
  input: "Mul_2142"
  input: "Mul_2143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_394"
  op: "Sqrt"
  input: "add_1343"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1344/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1344"
  op: "AddV2"
  input: "Sqrt_394"
  input: "add_1344/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_397"
  op: "RealDiv"
  input: "add_1342"
  input: "add_1344"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1724"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2144/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2144"
  op: "Mul"
  input: "mul_2144/x"
  input: "ReadVariableOp_1724"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1345"
  op: "AddV2"
  input: "truediv_397"
  input: "mul_2144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2145"
  op: "Mul"
  input: "mul_21"
  input: "add_1345"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1725"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_398"
  op: "Sub"
  input: "ReadVariableOp_1725"
  input: "mul_2145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1182"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  input: "sub_398"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1726"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/MatMul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1182"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1183"
  op: "AssignVariableOp"
  input: "Mul_2140/ReadVariableOp/Enter"
  input: "add_1342"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1727"
  op: "ReadVariableOp"
  input: "Mul_2140/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1183"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1184"
  op: "AssignVariableOp"
  input: "Mul_2142/ReadVariableOp/Enter"
  input: "add_1343"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1728"
  op: "ReadVariableOp"
  input: "Mul_2142/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1184"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/dense/bias/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_m"
  input: "electra/encoder/layer_11/output/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/dense/bias/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_v"
  input: "electra/encoder/layer_11/output/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2146/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2146/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2146/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2146/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2146"
  op: "Mul"
  input: "Mul_2146/x"
  input: "Mul_2146/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2147/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2147"
  op: "Mul"
  input: "Mul_2147/x"
  input: "CrossReplicaSum_395"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1346"
  op: "AddV2"
  input: "Mul_2146"
  input: "Mul_2147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2148/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2148/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2148/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2148/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2148"
  op: "Mul"
  input: "Mul_2148/x"
  input: "Mul_2148/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_395"
  op: "Square"
  input: "CrossReplicaSum_395"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2149/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2149"
  op: "Mul"
  input: "Mul_2149/x"
  input: "Square_395"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1347"
  op: "AddV2"
  input: "Mul_2148"
  input: "Mul_2149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_395"
  op: "Sqrt"
  input: "add_1347"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1348/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1348"
  op: "AddV2"
  input: "Sqrt_395"
  input: "add_1348/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_398"
  op: "RealDiv"
  input: "add_1346"
  input: "add_1348"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2150"
  op: "Mul"
  input: "mul_21"
  input: "truediv_398"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1729"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_399"
  op: "Sub"
  input: "ReadVariableOp_1729"
  input: "mul_2150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1185"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_399"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1730"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1185"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1186"
  op: "AssignVariableOp"
  input: "Mul_2146/ReadVariableOp/Enter"
  input: "add_1346"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1731"
  op: "ReadVariableOp"
  input: "Mul_2146/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1186"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1187"
  op: "AssignVariableOp"
  input: "Mul_2148/ReadVariableOp/Enter"
  input: "add_1347"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1732"
  op: "ReadVariableOp"
  input: "Mul_2148/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1187"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2151/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2151/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2151/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2151/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2151"
  op: "Mul"
  input: "Mul_2151/x"
  input: "Mul_2151/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2152/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2152"
  op: "Mul"
  input: "Mul_2152/x"
  input: "CrossReplicaSum_396"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1349"
  op: "AddV2"
  input: "Mul_2151"
  input: "Mul_2152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2153/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2153/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2153/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2153/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2153"
  op: "Mul"
  input: "Mul_2153/x"
  input: "Mul_2153/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_396"
  op: "Square"
  input: "CrossReplicaSum_396"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2154/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2154"
  op: "Mul"
  input: "Mul_2154/x"
  input: "Square_396"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1350"
  op: "AddV2"
  input: "Mul_2153"
  input: "Mul_2154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_396"
  op: "Sqrt"
  input: "add_1350"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1351/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1351"
  op: "AddV2"
  input: "Sqrt_396"
  input: "add_1351/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_399"
  op: "RealDiv"
  input: "add_1349"
  input: "add_1351"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2155"
  op: "Mul"
  input: "mul_21"
  input: "truediv_399"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1733"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_400"
  op: "Sub"
  input: "ReadVariableOp_1733"
  input: "mul_2155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1188"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "sub_400"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1734"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/mul/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1188"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1189"
  op: "AssignVariableOp"
  input: "Mul_2151/ReadVariableOp/Enter"
  input: "add_1349"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1735"
  op: "ReadVariableOp"
  input: "Mul_2151/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1189"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1190"
  op: "AssignVariableOp"
  input: "Mul_2153/ReadVariableOp/Enter"
  input: "add_1350"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1736"
  op: "ReadVariableOp"
  input: "Mul_2153/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1190"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v/Assign"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2156/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2156/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2156/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2156/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2156"
  op: "Mul"
  input: "Mul_2156/x"
  input: "Mul_2156/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2157/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2157"
  op: "Mul"
  input: "Mul_2157/x"
  input: "CrossReplicaSum_397"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1352"
  op: "AddV2"
  input: "Mul_2156"
  input: "Mul_2157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2158/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2158/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2158/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2158/ReadVariableOp/Enter"
  op: "Enter"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2158"
  op: "Mul"
  input: "Mul_2158/x"
  input: "Mul_2158/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_397"
  op: "Square"
  input: "CrossReplicaSum_397"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2159/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2159"
  op: "Mul"
  input: "Mul_2159/x"
  input: "Square_397"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1353"
  op: "AddV2"
  input: "Mul_2158"
  input: "Mul_2159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_397"
  op: "Sqrt"
  input: "add_1353"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1354/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1354"
  op: "AddV2"
  input: "Sqrt_397"
  input: "add_1354/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_400"
  op: "RealDiv"
  input: "add_1352"
  input: "add_1354"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2160"
  op: "Mul"
  input: "mul_21"
  input: "truediv_400"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1737"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_401"
  op: "Sub"
  input: "ReadVariableOp_1737"
  input: "mul_2160"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1191"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "sub_401"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1738"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/batchnorm/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1191"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1192"
  op: "AssignVariableOp"
  input: "Mul_2156/ReadVariableOp/Enter"
  input: "add_1352"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1739"
  op: "ReadVariableOp"
  input: "Mul_2156/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1192"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1193"
  op: "AssignVariableOp"
  input: "Mul_2158/ReadVariableOp/Enter"
  input: "add_1353"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1740"
  op: "ReadVariableOp"
  input: "Mul_2158/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1193"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_m/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_m/Initializer/zeros"
  op: "Fill"
  input: "discriminator_predictions/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor"
  input: "discriminator_predictions/dense/kernel/adam_m/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense/kernel/adam_m"
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/kernel/adam_m"
  input: "discriminator_predictions/dense/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_v/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_v/Initializer/zeros"
  op: "Fill"
  input: "discriminator_predictions/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor"
  input: "discriminator_predictions/dense/kernel/adam_v/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense/kernel/adam_v"
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/kernel/adam_v"
  input: "discriminator_predictions/dense/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2161/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2161/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2161/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2161/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2161"
  op: "Mul"
  input: "Mul_2161/x"
  input: "Mul_2161/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2162/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2162"
  op: "Mul"
  input: "Mul_2162/x"
  input: "CrossReplicaSum_398"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1355"
  op: "AddV2"
  input: "Mul_2161"
  input: "Mul_2162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2163/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2163/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2163/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2163/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2163"
  op: "Mul"
  input: "Mul_2163/x"
  input: "Mul_2163/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_398"
  op: "Square"
  input: "CrossReplicaSum_398"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2164/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2164"
  op: "Mul"
  input: "Mul_2164/x"
  input: "Square_398"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1356"
  op: "AddV2"
  input: "Mul_2163"
  input: "Mul_2164"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_398"
  op: "Sqrt"
  input: "add_1356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1357/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1357"
  op: "AddV2"
  input: "Sqrt_398"
  input: "add_1357/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_401"
  op: "RealDiv"
  input: "add_1355"
  input: "add_1357"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1741"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2165/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2165"
  op: "Mul"
  input: "mul_2165/x"
  input: "ReadVariableOp_1741"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1358"
  op: "AddV2"
  input: "truediv_401"
  input: "mul_2165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2166"
  op: "Mul"
  input: "mul_21"
  input: "add_1358"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1742"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_402"
  op: "Sub"
  input: "ReadVariableOp_1742"
  input: "mul_2166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1194"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  input: "sub_402"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1743"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/Tensordot/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1194"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1195"
  op: "AssignVariableOp"
  input: "Mul_2161/ReadVariableOp/Enter"
  input: "add_1355"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1744"
  op: "ReadVariableOp"
  input: "Mul_2161/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1195"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1196"
  op: "AssignVariableOp"
  input: "Mul_2163/ReadVariableOp/Enter"
  input: "add_1356"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1745"
  op: "ReadVariableOp"
  input: "Mul_2163/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1196"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense/bias/adam_m"
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/bias/adam_m"
  input: "discriminator_predictions/dense/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense/bias/adam_v"
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/bias/adam_v"
  input: "discriminator_predictions/dense/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2167/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2167/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2167/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2167/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2167"
  op: "Mul"
  input: "Mul_2167/x"
  input: "Mul_2167/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2168/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2168"
  op: "Mul"
  input: "Mul_2168/x"
  input: "CrossReplicaSum_399"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1359"
  op: "AddV2"
  input: "Mul_2167"
  input: "Mul_2168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2169/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2169/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2169/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2169/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2169"
  op: "Mul"
  input: "Mul_2169/x"
  input: "Mul_2169/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_399"
  op: "Square"
  input: "CrossReplicaSum_399"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2170/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2170"
  op: "Mul"
  input: "Mul_2170/x"
  input: "Square_399"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1360"
  op: "AddV2"
  input: "Mul_2169"
  input: "Mul_2170"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_399"
  op: "Sqrt"
  input: "add_1360"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1361/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1361"
  op: "AddV2"
  input: "Sqrt_399"
  input: "add_1361/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_402"
  op: "RealDiv"
  input: "add_1359"
  input: "add_1361"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2171"
  op: "Mul"
  input: "mul_21"
  input: "truediv_402"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1746"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_403"
  op: "Sub"
  input: "ReadVariableOp_1746"
  input: "mul_2171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1197"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/BiasAdd/ReadVariableOp/Enter"
  input: "sub_403"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1747"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1197"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1198"
  op: "AssignVariableOp"
  input: "Mul_2167/ReadVariableOp/Enter"
  input: "add_1359"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1748"
  op: "ReadVariableOp"
  input: "Mul_2167/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1198"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1199"
  op: "AssignVariableOp"
  input: "Mul_2169/ReadVariableOp/Enter"
  input: "add_1360"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1749"
  op: "ReadVariableOp"
  input: "Mul_2169/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1199"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 1
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense_1/kernel/adam_m"
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/adam_m/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/kernel/adam_m"
  input: "discriminator_predictions/dense_1/kernel/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/kernel/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 1
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense_1/kernel/adam_v"
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/adam_v/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/kernel/adam_v"
  input: "discriminator_predictions/dense_1/kernel/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/kernel/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2172/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2172/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2172/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2172/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense_1/kernel/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2172"
  op: "Mul"
  input: "Mul_2172/x"
  input: "Mul_2172/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2173/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2173"
  op: "Mul"
  input: "Mul_2173/x"
  input: "CrossReplicaSum_400"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1362"
  op: "AddV2"
  input: "Mul_2172"
  input: "Mul_2173"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2174/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2174/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2174/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2174/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense_1/kernel/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2174"
  op: "Mul"
  input: "Mul_2174/x"
  input: "Mul_2174/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_400"
  op: "Square"
  input: "CrossReplicaSum_400"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2175/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2175"
  op: "Mul"
  input: "Mul_2175/x"
  input: "Square_400"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1363"
  op: "AddV2"
  input: "Mul_2174"
  input: "Mul_2175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_400"
  op: "Sqrt"
  input: "add_1363"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1364/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1364"
  op: "AddV2"
  input: "Sqrt_400"
  input: "add_1364/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_403"
  op: "RealDiv"
  input: "add_1362"
  input: "add_1364"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1750"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "mul_2176/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.009999999776482582
      }
    }
  }
}
node {
  name: "mul_2176"
  op: "Mul"
  input: "mul_2176/x"
  input: "ReadVariableOp_1750"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1365"
  op: "AddV2"
  input: "truediv_403"
  input: "mul_2176"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2177"
  op: "Mul"
  input: "mul_21"
  input: "add_1365"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1751"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/Tensordot/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_404"
  op: "Sub"
  input: "ReadVariableOp_1751"
  input: "mul_2177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1200"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/Tensordot/ReadVariableOp/Enter"
  input: "sub_404"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1752"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/Tensordot/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1200"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1201"
  op: "AssignVariableOp"
  input: "Mul_2172/ReadVariableOp/Enter"
  input: "add_1362"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1753"
  op: "ReadVariableOp"
  input: "Mul_2172/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1201"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1202"
  op: "AssignVariableOp"
  input: "Mul_2174/ReadVariableOp/Enter"
  input: "add_1363"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1754"
  op: "ReadVariableOp"
  input: "Mul_2174/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1202"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/adam_m/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 1
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/adam_m"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/bias/adam_m"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense_1/bias/adam_m"
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/adam_m/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/adam_m/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/bias/adam_m"
  input: "discriminator_predictions/dense_1/bias/adam_m/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/adam_m/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/adam_v/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 1
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/adam_v"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@discriminator_predictions/dense_1/bias/adam_v"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "allowed_devices"
    value {
      list {
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 1
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "discriminator_predictions/dense_1/bias/adam_v"
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/adam_v/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/adam_v/Assign"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/bias/adam_v"
  input: "discriminator_predictions/dense_1/bias/adam_v/Initializer/zeros"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "discriminator_predictions/dense_1/bias/adam_v/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2178/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "Mul_2178/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2178/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2178/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense_1/bias/adam_m"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2178"
  op: "Mul"
  input: "Mul_2178/x"
  input: "Mul_2178/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2179/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.10000000149011612
      }
    }
  }
}
node {
  name: "Mul_2179"
  op: "Mul"
  input: "Mul_2179/x"
  input: "CrossReplicaSum_401"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1366"
  op: "AddV2"
  input: "Mul_2178"
  input: "Mul_2179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2180/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.9990000128746033
      }
    }
  }
}
node {
  name: "Mul_2180/ReadVariableOp"
  op: "ReadVariableOp"
  input: "Mul_2180/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Mul_2180/ReadVariableOp/Enter"
  op: "Enter"
  input: "discriminator_predictions/dense_1/bias/adam_v"
  input: "^TPUReplicateMetadata"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "frame_name"
    value {
      s: "while_context"
    }
  }
  attr {
    key: "is_constant"
    value {
      b: true
    }
  }
  attr {
    key: "parallel_iterations"
    value {
      i: 1
    }
  }
}
node {
  name: "Mul_2180"
  op: "Mul"
  input: "Mul_2180/x"
  input: "Mul_2180/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Square_401"
  op: "Square"
  input: "CrossReplicaSum_401"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Mul_2181/x"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0010000000474974513
      }
    }
  }
}
node {
  name: "Mul_2181"
  op: "Mul"
  input: "Mul_2181/x"
  input: "Square_401"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1367"
  op: "AddV2"
  input: "Mul_2180"
  input: "Mul_2181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Sqrt_401"
  op: "Sqrt"
  input: "add_1367"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "add_1368/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999974752427e-07
      }
    }
  }
}
node {
  name: "add_1368"
  op: "AddV2"
  input: "Sqrt_401"
  input: "add_1368/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "truediv_404"
  op: "RealDiv"
  input: "add_1366"
  input: "add_1368"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "mul_2182"
  op: "Mul"
  input: "mul_21"
  input: "truediv_404"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1755"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/BiasAdd/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "sub_405"
  op: "Sub"
  input: "ReadVariableOp_1755"
  input: "mul_2182"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1203"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/BiasAdd/ReadVariableOp/Enter"
  input: "sub_405"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1756"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/BiasAdd/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1203"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1204"
  op: "AssignVariableOp"
  input: "Mul_2178/ReadVariableOp/Enter"
  input: "add_1366"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1757"
  op: "ReadVariableOp"
  input: "Mul_2178/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1204"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "AssignVariableOp_1205"
  op: "AssignVariableOp"
  input: "Mul_2180/ReadVariableOp/Enter"
  input: "add_1367"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "ReadVariableOp_1758"
  op: "ReadVariableOp"
  input: "Mul_2180/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1205"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "group_deps"
  op: "NoOp"
  input: "^AssignVariableOp"
  input: "^AssignVariableOp_1"
  input: "^AssignVariableOp_10"
  input: "^AssignVariableOp_100"
  input: "^AssignVariableOp_1000"
  input: "^AssignVariableOp_1001"
  input: "^AssignVariableOp_1002"
  input: "^AssignVariableOp_1003"
  input: "^AssignVariableOp_1004"
  input: "^AssignVariableOp_1005"
  input: "^AssignVariableOp_1006"
  input: "^AssignVariableOp_1007"
  input: "^AssignVariableOp_1008"
  input: "^AssignVariableOp_1009"
  input: "^AssignVariableOp_101"
  input: "^AssignVariableOp_1010"
  input: "^AssignVariableOp_1011"
  input: "^AssignVariableOp_1012"
  input: "^AssignVariableOp_1013"
  input: "^AssignVariableOp_1014"
  input: "^AssignVariableOp_1015"
  input: "^AssignVariableOp_1016"
  input: "^AssignVariableOp_1017"
  input: "^AssignVariableOp_1018"
  input: "^AssignVariableOp_1019"
  input: "^AssignVariableOp_102"
  input: "^AssignVariableOp_1020"
  input: "^AssignVariableOp_1021"
  input: "^AssignVariableOp_1022"
  input: "^AssignVariableOp_1023"
  input: "^AssignVariableOp_1024"
  input: "^AssignVariableOp_1025"
  input: "^AssignVariableOp_1026"
  input: "^AssignVariableOp_1027"
  input: "^AssignVariableOp_1028"
  input: "^AssignVariableOp_1029"
  input: "^AssignVariableOp_103"
  input: "^AssignVariableOp_1030"
  input: "^AssignVariableOp_1031"
  input: "^AssignVariableOp_1032"
  input: "^AssignVariableOp_1033"
  input: "^AssignVariableOp_1034"
  input: "^AssignVariableOp_1035"
  input: "^AssignVariableOp_1036"
  input: "^AssignVariableOp_1037"
  input: "^AssignVariableOp_1038"
  input: "^AssignVariableOp_1039"
  input: "^AssignVariableOp_104"
  input: "^AssignVariableOp_1040"
  input: "^AssignVariableOp_1041"
  input: "^AssignVariableOp_1042"
  input: "^AssignVariableOp_1043"
  input: "^AssignVariableOp_1044"
  input: "^AssignVariableOp_1045"
  input: "^AssignVariableOp_1046"
  input: "^AssignVariableOp_1047"
  input: "^AssignVariableOp_1048"
  input: "^AssignVariableOp_1049"
  input: "^AssignVariableOp_105"
  input: "^AssignVariableOp_1050"
  input: "^AssignVariableOp_1051"
  input: "^AssignVariableOp_1052"
  input: "^AssignVariableOp_1053"
  input: "^AssignVariableOp_1054"
  input: "^AssignVariableOp_1055"
  input: "^AssignVariableOp_1056"
  input: "^AssignVariableOp_1057"
  input: "^AssignVariableOp_1058"
  input: "^AssignVariableOp_1059"
  input: "^AssignVariableOp_106"
  input: "^AssignVariableOp_1060"
  input: "^AssignVariableOp_1061"
  input: "^AssignVariableOp_1062"
  input: "^AssignVariableOp_1063"
  input: "^AssignVariableOp_1064"
  input: "^AssignVariableOp_1065"
  input: "^AssignVariableOp_1066"
  input: "^AssignVariableOp_1067"
  input: "^AssignVariableOp_1068"
  input: "^AssignVariableOp_1069"
  input: "^AssignVariableOp_107"
  input: "^AssignVariableOp_1070"
  input: "^AssignVariableOp_1071"
  input: "^AssignVariableOp_1072"
  input: "^AssignVariableOp_1073"
  input: "^AssignVariableOp_1074"
  input: "^AssignVariableOp_1075"
  input: "^AssignVariableOp_1076"
  input: "^AssignVariableOp_1077"
  input: "^AssignVariableOp_1078"
  input: "^AssignVariableOp_1079"
  input: "^AssignVariableOp_108"
  input: "^AssignVariableOp_1080"
  input: "^AssignVariableOp_1081"
  input: "^AssignVariableOp_1082"
  input: "^AssignVariableOp_1083"
  input: "^AssignVariableOp_1084"
  input: "^AssignVariableOp_1085"
  input: "^AssignVariableOp_1086"
  input: "^AssignVariableOp_1087"
  input: "^AssignVariableOp_1088"
  input: "^AssignVariableOp_1089"
  input: "^AssignVariableOp_109"
  input: "^AssignVariableOp_1090"
  input: "^AssignVariableOp_1091"
  input: "^AssignVariableOp_1092"
  input: "^AssignVariableOp_1093"
  input: "^AssignVariableOp_1094"
  input: "^AssignVariableOp_1095"
  input: "^AssignVariableOp_1096"
  input: "^AssignVariableOp_1097"
  input: "^AssignVariableOp_1098"
  input: "^AssignVariableOp_1099"
  input: "^AssignVariableOp_11"
  input: "^AssignVariableOp_110"
  input: "^AssignVariableOp_1100"
  input: "^AssignVariableOp_1101"
  input: "^AssignVariableOp_1102"
  input: "^AssignVariableOp_1103"
  input: "^AssignVariableOp_1104"
  input: "^AssignVariableOp_1105"
  input: "^AssignVariableOp_1106"
  input: "^AssignVariableOp_1107"
  input: "^AssignVariableOp_1108"
  input: "^AssignVariableOp_1109"
  input: "^AssignVariableOp_111"
  input: "^AssignVariableOp_1110"
  input: "^AssignVariableOp_1111"
  input: "^AssignVariableOp_1112"
  input: "^AssignVariableOp_1113"
  input: "^AssignVariableOp_1114"
  input: "^AssignVariableOp_1115"
  input: "^AssignVariableOp_1116"
  input: "^AssignVariableOp_1117"
  input: "^AssignVariableOp_1118"
  input: "^AssignVariableOp_1119"
  input: "^AssignVariableOp_112"
  input: "^AssignVariableOp_1120"
  input: "^AssignVariableOp_1121"
  input: "^AssignVariableOp_1122"
  input: "^AssignVariableOp_1123"
  input: "^AssignVariableOp_1124"
  input: "^AssignVariableOp_1125"
  input: "^AssignVariableOp_1126"
  input: "^AssignVariableOp_1127"
  input: "^AssignVariableOp_1128"
  input: "^AssignVariableOp_1129"
  input: "^AssignVariableOp_113"
  input: "^AssignVariableOp_1130"
  input: "^AssignVariableOp_1131"
  input: "^AssignVariableOp_1132"
  input: "^AssignVariableOp_1133"
  input: "^AssignVariableOp_1134"
  input: "^AssignVariableOp_1135"
  input: "^AssignVariableOp_1136"
  input: "^AssignVariableOp_1137"
  input: "^AssignVariableOp_1138"
  input: "^AssignVariableOp_1139"
  input: "^AssignVariableOp_114"
  input: "^AssignVariableOp_1140"
  input: "^AssignVariableOp_1141"
  input: "^AssignVariableOp_1142"
  input: "^AssignVariableOp_1143"
  input: "^AssignVariableOp_1144"
  input: "^AssignVariableOp_1145"
  input: "^AssignVariableOp_1146"
  input: "^AssignVariableOp_1147"
  input: "^AssignVariableOp_1148"
  input: "^AssignVariableOp_1149"
  input: "^AssignVariableOp_115"
  input: "^AssignVariableOp_1150"
  input: "^AssignVariableOp_1151"
  input: "^AssignVariableOp_1152"
  input: "^AssignVariableOp_1153"
  input: "^AssignVariableOp_1154"
  input: "^AssignVariableOp_1155"
  input: "^AssignVariableOp_1156"
  input: "^AssignVariableOp_1157"
  input: "^AssignVariableOp_1158"
  input: "^AssignVariableOp_1159"
  input: "^AssignVariableOp_116"
  input: "^AssignVariableOp_1160"
  input: "^AssignVariableOp_1161"
  input: "^AssignVariableOp_1162"
  input: "^AssignVariableOp_1163"
  input: "^AssignVariableOp_1164"
  input: "^AssignVariableOp_1165"
  input: "^AssignVariableOp_1166"
  input: "^AssignVariableOp_1167"
  input: "^AssignVariableOp_1168"
  input: "^AssignVariableOp_1169"
  input: "^AssignVariableOp_117"
  input: "^AssignVariableOp_1170"
  input: "^AssignVariableOp_1171"
  input: "^AssignVariableOp_1172"
  input: "^AssignVariableOp_1173"
  input: "^AssignVariableOp_1174"
  input: "^AssignVariableOp_1175"
  input: "^AssignVariableOp_1176"
  input: "^AssignVariableOp_1177"
  input: "^AssignVariableOp_1178"
  input: "^AssignVariableOp_1179"
  input: "^AssignVariableOp_118"
  input: "^AssignVariableOp_1180"
  input: "^AssignVariableOp_1181"
  input: "^AssignVariableOp_1182"
  input: "^AssignVariableOp_1183"
  input: "^AssignVariableOp_1184"
  input: "^AssignVariableOp_1185"
  input: "^AssignVariableOp_1186"
  input: "^AssignVariableOp_1187"
  input: "^AssignVariableOp_1188"
  input: "^AssignVariableOp_1189"
  input: "^AssignVariableOp_119"
  input: "^AssignVariableOp_1190"
  input: "^AssignVariableOp_1191"
  input: "^AssignVariableOp_1192"
  input: "^AssignVariableOp_1193"
  input: "^AssignVariableOp_1194"
  input: "^AssignVariableOp_1195"
  input: "^AssignVariableOp_1196"
  input: "^AssignVariableOp_1197"
  input: "^AssignVariableOp_1198"
  input: "^AssignVariableOp_1199"
  input: "^AssignVariableOp_12"
  input: "^AssignVariableOp_120"
  input: "^AssignVariableOp_1200"
  input: "^AssignVariableOp_1201"
  input: "^AssignVariableOp_1202"
  input: "^AssignVariableOp_1203"
  input: "^AssignVariableOp_1204"
  input: "^AssignVariableOp_1205"
  input: "^AssignVariableOp_121"
  input: "^AssignVariableOp_122"
  input: "^AssignVariableOp_123"
  input: "^AssignVariableOp_124"
  input: "^AssignVariableOp_125"
  input: "^AssignVariableOp_126"
  input: "^AssignVariableOp_127"
  input: "^AssignVariableOp_128"
  input: "^AssignVariableOp_129"
  input: "^AssignVariableOp_13"
  input: "^AssignVariableOp_130"
  input: "^AssignVariableOp_131"
  input: "^AssignVariableOp_132"
  input: "^AssignVariableOp_133"
  input: "^AssignVariableOp_134"
  input: "^AssignVariableOp_135"
  input: "^AssignVariableOp_136"
  input: "^AssignVariableOp_137"
  input: "^AssignVariableOp_138"
  input: "^AssignVariableOp_139"
  input: "^AssignVariableOp_14"
  input: "^AssignVariableOp_140"
  input: "^AssignVariableOp_141"
  input: "^AssignVariableOp_142"
  input: "^AssignVariableOp_143"
  input: "^AssignVariableOp_144"
  input: "^AssignVariableOp_145"
  input: "^AssignVariableOp_146"
  input: "^AssignVariableOp_147"
  input: "^AssignVariableOp_148"
  input: "^AssignVariableOp_149"
  input: "^AssignVariableOp_15"
  input: "^AssignVariableOp_150"
  input: "^AssignVariableOp_151"
  input: "^AssignVariableOp_152"
  input: "^AssignVariableOp_153"
  input: "^AssignVariableOp_154"
  input: "^AssignVariableOp_155"
  input: "^AssignVariableOp_156"
  input: "^AssignVariableOp_157"
  input: "^AssignVariableOp_158"
  input: "^AssignVariableOp_159"
  input: "^AssignVariableOp_16"
  input: "^AssignVariableOp_160"
  input: "^AssignVariableOp_161"
  input: "^AssignVariableOp_162"
  input: "^AssignVariableOp_163"
  input: "^AssignVariableOp_164"
  input: "^AssignVariableOp_165"
  input: "^AssignVariableOp_166"
  input: "^AssignVariableOp_167"
  input: "^AssignVariableOp_168"
  input: "^AssignVariableOp_169"
  input: "^AssignVariableOp_17"
  input: "^AssignVariableOp_170"
  input: "^AssignVariableOp_171"
  input: "^AssignVariableOp_172"
  input: "^AssignVariableOp_173"
  input: "^AssignVariableOp_174"
  input: "^AssignVariableOp_175"
  input: "^AssignVariableOp_176"
  input: "^AssignVariableOp_177"
  input: "^AssignVariableOp_178"
  input: "^AssignVariableOp_179"
  input: "^AssignVariableOp_18"
  input: "^AssignVariableOp_180"
  input: "^AssignVariableOp_181"
  input: "^AssignVariableOp_182"
  input: "^AssignVariableOp_183"
  input: "^AssignVariableOp_184"
  input: "^AssignVariableOp_185"
  input: "^AssignVariableOp_186"
  input: "^AssignVariableOp_187"
  input: "^AssignVariableOp_188"
  input: "^AssignVariableOp_189"
  input: "^AssignVariableOp_19"
  input: "^AssignVariableOp_190"
  input: "^AssignVariableOp_191"
  input: "^AssignVariableOp_192"
  input: "^AssignVariableOp_193"
  input: "^AssignVariableOp_194"
  input: "^AssignVariableOp_195"
  input: "^AssignVariableOp_196"
  input: "^AssignVariableOp_197"
  input: "^AssignVariableOp_198"
  input: "^AssignVariableOp_199"
  input: "^AssignVariableOp_2"
  input: "^AssignVariableOp_20"
  input: "^AssignVariableOp_200"
  input: "^AssignVariableOp_201"
  input: "^AssignVariableOp_202"
  input: "^AssignVariableOp_203"
  input: "^AssignVariableOp_204"
  input: "^AssignVariableOp_205"
  input: "^AssignVariableOp_206"
  input: "^AssignVariableOp_207"
  input: "^AssignVariableOp_208"
  input: "^AssignVariableOp_209"
  input: "^AssignVariableOp_21"
  input: "^AssignVariableOp_210"
  input: "^AssignVariableOp_211"
  input: "^AssignVariableOp_212"
  input: "^AssignVariableOp_213"
  input: "^AssignVariableOp_214"
  input: "^AssignVariableOp_215"
  input: "^AssignVariableOp_216"
  input: "^AssignVariableOp_217"
  input: "^AssignVariableOp_218"
  input: "^AssignVariableOp_219"
  input: "^AssignVariableOp_22"
  input: "^AssignVariableOp_220"
  input: "^AssignVariableOp_221"
  input: "^AssignVariableOp_222"
  input: "^AssignVariableOp_223"
  input: "^AssignVariableOp_224"
  input: "^AssignVariableOp_225"
  input: "^AssignVariableOp_226"
  input: "^AssignVariableOp_227"
  input: "^AssignVariableOp_228"
  input: "^AssignVariableOp_229"
  input: "^AssignVariableOp_23"
  input: "^AssignVariableOp_230"
  input: "^AssignVariableOp_231"
  input: "^AssignVariableOp_232"
  input: "^AssignVariableOp_233"
  input: "^AssignVariableOp_234"
  input: "^AssignVariableOp_235"
  input: "^AssignVariableOp_236"
  input: "^AssignVariableOp_237"
  input: "^AssignVariableOp_238"
  input: "^AssignVariableOp_239"
  input: "^AssignVariableOp_24"
  input: "^AssignVariableOp_240"
  input: "^AssignVariableOp_241"
  input: "^AssignVariableOp_242"
  input: "^AssignVariableOp_243"
  input: "^AssignVariableOp_244"
  input: "^AssignVariableOp_245"
  input: "^AssignVariableOp_246"
  input: "^AssignVariableOp_247"
  input: "^AssignVariableOp_248"
  input: "^AssignVariableOp_249"
  input: "^AssignVariableOp_25"
  input: "^AssignVariableOp_250"
  input: "^AssignVariableOp_251"
  input: "^AssignVariableOp_252"
  input: "^AssignVariableOp_253"
  input: "^AssignVariableOp_254"
  input: "^AssignVariableOp_255"
  input: "^AssignVariableOp_256"
  input: "^AssignVariableOp_257"
  input: "^AssignVariableOp_258"
  input: "^AssignVariableOp_259"
  input: "^AssignVariableOp_26"
  input: "^AssignVariableOp_260"
  input: "^AssignVariableOp_261"
  input: "^AssignVariableOp_262"
  input: "^AssignVariableOp_263"
  input: "^AssignVariableOp_264"
  input: "^AssignVariableOp_265"
  input: "^AssignVariableOp_266"
  input: "^AssignVariableOp_267"
  input: "^AssignVariableOp_268"
  input: "^AssignVariableOp_269"
  input: "^AssignVariableOp_27"
  input: "^AssignVariableOp_270"
  input: "^AssignVariableOp_271"
  input: "^AssignVariableOp_272"
  input: "^AssignVariableOp_273"
  input: "^AssignVariableOp_274"
  input: "^AssignVariableOp_275"
  input: "^AssignVariableOp_276"
  input: "^AssignVariableOp_277"
  input: "^AssignVariableOp_278"
  input: "^AssignVariableOp_279"
  input: "^AssignVariableOp_28"
  input: "^AssignVariableOp_280"
  input: "^AssignVariableOp_281"
  input: "^AssignVariableOp_282"
  input: "^AssignVariableOp_283"
  input: "^AssignVariableOp_284"
  input: "^AssignVariableOp_285"
  input: "^AssignVariableOp_286"
  input: "^AssignVariableOp_287"
  input: "^AssignVariableOp_288"
  input: "^AssignVariableOp_289"
  input: "^AssignVariableOp_29"
  input: "^AssignVariableOp_290"
  input: "^AssignVariableOp_291"
  input: "^AssignVariableOp_292"
  input: "^AssignVariableOp_293"
  input: "^AssignVariableOp_294"
  input: "^AssignVariableOp_295"
  input: "^AssignVariableOp_296"
  input: "^AssignVariableOp_297"
  input: "^AssignVariableOp_298"
  input: "^AssignVariableOp_299"
  input: "^AssignVariableOp_3"
  input: "^AssignVariableOp_30"
  input: "^AssignVariableOp_300"
  input: "^AssignVariableOp_301"
  input: "^AssignVariableOp_302"
  input: "^AssignVariableOp_303"
  input: "^AssignVariableOp_304"
  input: "^AssignVariableOp_305"
  input: "^AssignVariableOp_306"
  input: "^AssignVariableOp_307"
  input: "^AssignVariableOp_308"
  input: "^AssignVariableOp_309"
  input: "^AssignVariableOp_31"
  input: "^AssignVariableOp_310"
  input: "^AssignVariableOp_311"
  input: "^AssignVariableOp_312"
  input: "^AssignVariableOp_313"
  input: "^AssignVariableOp_314"
  input: "^AssignVariableOp_315"
  input: "^AssignVariableOp_316"
  input: "^AssignVariableOp_317"
  input: "^AssignVariableOp_318"
  input: "^AssignVariableOp_319"
  input: "^AssignVariableOp_32"
  input: "^AssignVariableOp_320"
  input: "^AssignVariableOp_321"
  input: "^AssignVariableOp_322"
  input: "^AssignVariableOp_323"
  input: "^AssignVariableOp_324"
  input: "^AssignVariableOp_325"
  input: "^AssignVariableOp_326"
  input: "^AssignVariableOp_327"
  input: "^AssignVariableOp_328"
  input: "^AssignVariableOp_329"
  input: "^AssignVariableOp_33"
  input: "^AssignVariableOp_330"
  input: "^AssignVariableOp_331"
  input: "^AssignVariableOp_332"
  input: "^AssignVariableOp_333"
  input: "^AssignVariableOp_334"
  input: "^AssignVariableOp_335"
  input: "^AssignVariableOp_336"
  input: "^AssignVariableOp_337"
  input: "^AssignVariableOp_338"
  input: "^AssignVariableOp_339"
  input: "^AssignVariableOp_34"
  input: "^AssignVariableOp_340"
  input: "^AssignVariableOp_341"
  input: "^AssignVariableOp_342"
  input: "^AssignVariableOp_343"
  input: "^AssignVariableOp_344"
  input: "^AssignVariableOp_345"
  input: "^AssignVariableOp_346"
  input: "^AssignVariableOp_347"
  input: "^AssignVariableOp_348"
  input: "^AssignVariableOp_349"
  input: "^AssignVariableOp_35"
  input: "^AssignVariableOp_350"
  input: "^AssignVariableOp_351"
  input: "^AssignVariableOp_352"
  input: "^AssignVariableOp_353"
  input: "^AssignVariableOp_354"
  input: "^AssignVariableOp_355"
  input: "^AssignVariableOp_356"
  input: "^AssignVariableOp_357"
  input: "^AssignVariableOp_358"
  input: "^AssignVariableOp_359"
  input: "^AssignVariableOp_36"
  input: "^AssignVariableOp_360"
  input: "^AssignVariableOp_361"
  input: "^AssignVariableOp_362"
  input: "^AssignVariableOp_363"
  input: "^AssignVariableOp_364"
  input: "^AssignVariableOp_365"
  input: "^AssignVariableOp_366"
  input: "^AssignVariableOp_367"
  input: "^AssignVariableOp_368"
  input: "^AssignVariableOp_369"
  input: "^AssignVariableOp_37"
  input: "^AssignVariableOp_370"
  input: "^AssignVariableOp_371"
  input: "^AssignVariableOp_372"
  input: "^AssignVariableOp_373"
  input: "^AssignVariableOp_374"
  input: "^AssignVariableOp_375"
  input: "^AssignVariableOp_376"
  input: "^AssignVariableOp_377"
  input: "^AssignVariableOp_378"
  input: "^AssignVariableOp_379"
  input: "^AssignVariableOp_38"
  input: "^AssignVariableOp_380"
  input: "^AssignVariableOp_381"
  input: "^AssignVariableOp_382"
  input: "^AssignVariableOp_383"
  input: "^AssignVariableOp_384"
  input: "^AssignVariableOp_385"
  input: "^AssignVariableOp_386"
  input: "^AssignVariableOp_387"
  input: "^AssignVariableOp_388"
  input: "^AssignVariableOp_389"
  input: "^AssignVariableOp_39"
  input: "^AssignVariableOp_390"
  input: "^AssignVariableOp_391"
  input: "^AssignVariableOp_392"
  input: "^AssignVariableOp_393"
  input: "^AssignVariableOp_394"
  input: "^AssignVariableOp_395"
  input: "^AssignVariableOp_396"
  input: "^AssignVariableOp_397"
  input: "^AssignVariableOp_398"
  input: "^AssignVariableOp_399"
  input: "^AssignVariableOp_4"
  input: "^AssignVariableOp_40"
  input: "^AssignVariableOp_400"
  input: "^AssignVariableOp_401"
  input: "^AssignVariableOp_402"
  input: "^AssignVariableOp_403"
  input: "^AssignVariableOp_404"
  input: "^AssignVariableOp_405"
  input: "^AssignVariableOp_406"
  input: "^AssignVariableOp_407"
  input: "^AssignVariableOp_408"
  input: "^AssignVariableOp_409"
  input: "^AssignVariableOp_41"
  input: "^AssignVariableOp_410"
  input: "^AssignVariableOp_411"
  input: "^AssignVariableOp_412"
  input: "^AssignVariableOp_413"
  input: "^AssignVariableOp_414"
  input: "^AssignVariableOp_415"
  input: "^AssignVariableOp_416"
  input: "^AssignVariableOp_417"
  input: "^AssignVariableOp_418"
  input: "^AssignVariableOp_419"
  input: "^AssignVariableOp_42"
  input: "^AssignVariableOp_420"
  input: "^AssignVariableOp_421"
  input: "^AssignVariableOp_422"
  input: "^AssignVariableOp_423"
  input: "^AssignVariableOp_424"
  input: "^AssignVariableOp_425"
  input: "^AssignVariableOp_426"
  input: "^AssignVariableOp_427"
  input: "^AssignVariableOp_428"
  input: "^AssignVariableOp_429"
  input: "^AssignVariableOp_43"
  input: "^AssignVariableOp_430"
  input: "^AssignVariableOp_431"
  input: "^AssignVariableOp_432"
  input: "^AssignVariableOp_433"
  input: "^AssignVariableOp_434"
  input: "^AssignVariableOp_435"
  input: "^AssignVariableOp_436"
  input: "^AssignVariableOp_437"
  input: "^AssignVariableOp_438"
  input: "^AssignVariableOp_439"
  input: "^AssignVariableOp_44"
  input: "^AssignVariableOp_440"
  input: "^AssignVariableOp_441"
  input: "^AssignVariableOp_442"
  input: "^AssignVariableOp_443"
  input: "^AssignVariableOp_444"
  input: "^AssignVariableOp_445"
  input: "^AssignVariableOp_446"
  input: "^AssignVariableOp_447"
  input: "^AssignVariableOp_448"
  input: "^AssignVariableOp_449"
  input: "^AssignVariableOp_45"
  input: "^AssignVariableOp_450"
  input: "^AssignVariableOp_451"
  input: "^AssignVariableOp_452"
  input: "^AssignVariableOp_453"
  input: "^AssignVariableOp_454"
  input: "^AssignVariableOp_455"
  input: "^AssignVariableOp_456"
  input: "^AssignVariableOp_457"
  input: "^AssignVariableOp_458"
  input: "^AssignVariableOp_459"
  input: "^AssignVariableOp_46"
  input: "^AssignVariableOp_460"
  input: "^AssignVariableOp_461"
  input: "^AssignVariableOp_462"
  input: "^AssignVariableOp_463"
  input: "^AssignVariableOp_464"
  input: "^AssignVariableOp_465"
  input: "^AssignVariableOp_466"
  input: "^AssignVariableOp_467"
  input: "^AssignVariableOp_468"
  input: "^AssignVariableOp_469"
  input: "^AssignVariableOp_47"
  input: "^AssignVariableOp_470"
  input: "^AssignVariableOp_471"
  input: "^AssignVariableOp_472"
  input: "^AssignVariableOp_473"
  input: "^AssignVariableOp_474"
  input: "^AssignVariableOp_475"
  input: "^AssignVariableOp_476"
  input: "^AssignVariableOp_477"
  input: "^AssignVariableOp_478"
  input: "^AssignVariableOp_479"
  input: "^AssignVariableOp_48"
  input: "^AssignVariableOp_480"
  input: "^AssignVariableOp_481"
  input: "^AssignVariableOp_482"
  input: "^AssignVariableOp_483"
  input: "^AssignVariableOp_484"
  input: "^AssignVariableOp_485"
  input: "^AssignVariableOp_486"
  input: "^AssignVariableOp_487"
  input: "^AssignVariableOp_488"
  input: "^AssignVariableOp_489"
  input: "^AssignVariableOp_49"
  input: "^AssignVariableOp_490"
  input: "^AssignVariableOp_491"
  input: "^AssignVariableOp_492"
  input: "^AssignVariableOp_493"
  input: "^AssignVariableOp_494"
  input: "^AssignVariableOp_495"
  input: "^AssignVariableOp_496"
  input: "^AssignVariableOp_497"
  input: "^AssignVariableOp_498"
  input: "^AssignVariableOp_499"
  input: "^AssignVariableOp_5"
  input: "^AssignVariableOp_50"
  input: "^AssignVariableOp_500"
  input: "^AssignVariableOp_501"
  input: "^AssignVariableOp_502"
  input: "^AssignVariableOp_503"
  input: "^AssignVariableOp_504"
  input: "^AssignVariableOp_505"
  input: "^AssignVariableOp_506"
  input: "^AssignVariableOp_507"
  input: "^AssignVariableOp_508"
  input: "^AssignVariableOp_509"
  input: "^AssignVariableOp_51"
  input: "^AssignVariableOp_510"
  input: "^AssignVariableOp_511"
  input: "^AssignVariableOp_512"
  input: "^AssignVariableOp_513"
  input: "^AssignVariableOp_514"
  input: "^AssignVariableOp_515"
  input: "^AssignVariableOp_516"
  input: "^AssignVariableOp_517"
  input: "^AssignVariableOp_518"
  input: "^AssignVariableOp_519"
  input: "^AssignVariableOp_52"
  input: "^AssignVariableOp_520"
  input: "^AssignVariableOp_521"
  input: "^AssignVariableOp_522"
  input: "^AssignVariableOp_523"
  input: "^AssignVariableOp_524"
  input: "^AssignVariableOp_525"
  input: "^AssignVariableOp_526"
  input: "^AssignVariableOp_527"
  input: "^AssignVariableOp_528"
  input: "^AssignVariableOp_529"
  input: "^AssignVariableOp_53"
  input: "^AssignVariableOp_530"
  input: "^AssignVariableOp_531"
  input: "^AssignVariableOp_532"
  input: "^AssignVariableOp_533"
  input: "^AssignVariableOp_534"
  input: "^AssignVariableOp_535"
  input: "^AssignVariableOp_536"
  input: "^AssignVariableOp_537"
  input: "^AssignVariableOp_538"
  input: "^AssignVariableOp_539"
  input: "^AssignVariableOp_54"
  input: "^AssignVariableOp_540"
  input: "^AssignVariableOp_541"
  input: "^AssignVariableOp_542"
  input: "^AssignVariableOp_543"
  input: "^AssignVariableOp_544"
  input: "^AssignVariableOp_545"
  input: "^AssignVariableOp_546"
  input: "^AssignVariableOp_547"
  input: "^AssignVariableOp_548"
  input: "^AssignVariableOp_549"
  input: "^AssignVariableOp_55"
  input: "^AssignVariableOp_550"
  input: "^AssignVariableOp_551"
  input: "^AssignVariableOp_552"
  input: "^AssignVariableOp_553"
  input: "^AssignVariableOp_554"
  input: "^AssignVariableOp_555"
  input: "^AssignVariableOp_556"
  input: "^AssignVariableOp_557"
  input: "^AssignVariableOp_558"
  input: "^AssignVariableOp_559"
  input: "^AssignVariableOp_56"
  input: "^AssignVariableOp_560"
  input: "^AssignVariableOp_561"
  input: "^AssignVariableOp_562"
  input: "^AssignVariableOp_563"
  input: "^AssignVariableOp_564"
  input: "^AssignVariableOp_565"
  input: "^AssignVariableOp_566"
  input: "^AssignVariableOp_567"
  input: "^AssignVariableOp_568"
  input: "^AssignVariableOp_569"
  input: "^AssignVariableOp_57"
  input: "^AssignVariableOp_570"
  input: "^AssignVariableOp_571"
  input: "^AssignVariableOp_572"
  input: "^AssignVariableOp_573"
  input: "^AssignVariableOp_574"
  input: "^AssignVariableOp_575"
  input: "^AssignVariableOp_576"
  input: "^AssignVariableOp_577"
  input: "^AssignVariableOp_578"
  input: "^AssignVariableOp_579"
  input: "^AssignVariableOp_58"
  input: "^AssignVariableOp_580"
  input: "^AssignVariableOp_581"
  input: "^AssignVariableOp_582"
  input: "^AssignVariableOp_583"
  input: "^AssignVariableOp_584"
  input: "^AssignVariableOp_585"
  input: "^AssignVariableOp_586"
  input: "^AssignVariableOp_587"
  input: "^AssignVariableOp_588"
  input: "^AssignVariableOp_589"
  input: "^AssignVariableOp_59"
  input: "^AssignVariableOp_590"
  input: "^AssignVariableOp_591"
  input: "^AssignVariableOp_592"
  input: "^AssignVariableOp_593"
  input: "^AssignVariableOp_594"
  input: "^AssignVariableOp_595"
  input: "^AssignVariableOp_596"
  input: "^AssignVariableOp_597"
  input: "^AssignVariableOp_598"
  input: "^AssignVariableOp_599"
  input: "^AssignVariableOp_6"
  input: "^AssignVariableOp_60"
  input: "^AssignVariableOp_600"
  input: "^AssignVariableOp_601"
  input: "^AssignVariableOp_602"
  input: "^AssignVariableOp_603"
  input: "^AssignVariableOp_604"
  input: "^AssignVariableOp_605"
  input: "^AssignVariableOp_606"
  input: "^AssignVariableOp_607"
  input: "^AssignVariableOp_608"
  input: "^AssignVariableOp_609"
  input: "^AssignVariableOp_61"
  input: "^AssignVariableOp_610"
  input: "^AssignVariableOp_611"
  input: "^AssignVariableOp_612"
  input: "^AssignVariableOp_613"
  input: "^AssignVariableOp_614"
  input: "^AssignVariableOp_615"
  input: "^AssignVariableOp_616"
  input: "^AssignVariableOp_617"
  input: "^AssignVariableOp_618"
  input: "^AssignVariableOp_619"
  input: "^AssignVariableOp_62"
  input: "^AssignVariableOp_620"
  input: "^AssignVariableOp_621"
  input: "^AssignVariableOp_622"
  input: "^AssignVariableOp_623"
  input: "^AssignVariableOp_624"
  input: "^AssignVariableOp_625"
  input: "^AssignVariableOp_626"
  input: "^AssignVariableOp_627"
  input: "^AssignVariableOp_628"
  input: "^AssignVariableOp_629"
  input: "^AssignVariableOp_63"
  input: "^AssignVariableOp_630"
  input: "^AssignVariableOp_631"
  input: "^AssignVariableOp_632"
  input: "^AssignVariableOp_633"
  input: "^AssignVariableOp_634"
  input: "^AssignVariableOp_635"
  input: "^AssignVariableOp_636"
  input: "^AssignVariableOp_637"
  input: "^AssignVariableOp_638"
  input: "^AssignVariableOp_639"
  input: "^AssignVariableOp_64"
  input: "^AssignVariableOp_640"
  input: "^AssignVariableOp_641"
  input: "^AssignVariableOp_642"
  input: "^AssignVariableOp_643"
  input: "^AssignVariableOp_644"
  input: "^AssignVariableOp_645"
  input: "^AssignVariableOp_646"
  input: "^AssignVariableOp_647"
  input: "^AssignVariableOp_648"
  input: "^AssignVariableOp_649"
  input: "^AssignVariableOp_65"
  input: "^AssignVariableOp_650"
  input: "^AssignVariableOp_651"
  input: "^AssignVariableOp_652"
  input: "^AssignVariableOp_653"
  input: "^AssignVariableOp_654"
  input: "^AssignVariableOp_655"
  input: "^AssignVariableOp_656"
  input: "^AssignVariableOp_657"
  input: "^AssignVariableOp_658"
  input: "^AssignVariableOp_659"
  input: "^AssignVariableOp_66"
  input: "^AssignVariableOp_660"
  input: "^AssignVariableOp_661"
  input: "^AssignVariableOp_662"
  input: "^AssignVariableOp_663"
  input: "^AssignVariableOp_664"
  input: "^AssignVariableOp_665"
  input: "^AssignVariableOp_666"
  input: "^AssignVariableOp_667"
  input: "^AssignVariableOp_668"
  input: "^AssignVariableOp_669"
  input: "^AssignVariableOp_67"
  input: "^AssignVariableOp_670"
  input: "^AssignVariableOp_671"
  input: "^AssignVariableOp_672"
  input: "^AssignVariableOp_673"
  input: "^AssignVariableOp_674"
  input: "^AssignVariableOp_675"
  input: "^AssignVariableOp_676"
  input: "^AssignVariableOp_677"
  input: "^AssignVariableOp_678"
  input: "^AssignVariableOp_679"
  input: "^AssignVariableOp_68"
  input: "^AssignVariableOp_680"
  input: "^AssignVariableOp_681"
  input: "^AssignVariableOp_682"
  input: "^AssignVariableOp_683"
  input: "^AssignVariableOp_684"
  input: "^AssignVariableOp_685"
  input: "^AssignVariableOp_686"
  input: "^AssignVariableOp_687"
  input: "^AssignVariableOp_688"
  input: "^AssignVariableOp_689"
  input: "^AssignVariableOp_69"
  input: "^AssignVariableOp_690"
  input: "^AssignVariableOp_691"
  input: "^AssignVariableOp_692"
  input: "^AssignVariableOp_693"
  input: "^AssignVariableOp_694"
  input: "^AssignVariableOp_695"
  input: "^AssignVariableOp_696"
  input: "^AssignVariableOp_697"
  input: "^AssignVariableOp_698"
  input: "^AssignVariableOp_699"
  input: "^AssignVariableOp_7"
  input: "^AssignVariableOp_70"
  input: "^AssignVariableOp_700"
  input: "^AssignVariableOp_701"
  input: "^AssignVariableOp_702"
  input: "^AssignVariableOp_703"
  input: "^AssignVariableOp_704"
  input: "^AssignVariableOp_705"
  input: "^AssignVariableOp_706"
  input: "^AssignVariableOp_707"
  input: "^AssignVariableOp_708"
  input: "^AssignVariableOp_709"
  input: "^AssignVariableOp_71"
  input: "^AssignVariableOp_710"
  input: "^AssignVariableOp_711"
  input: "^AssignVariableOp_712"
  input: "^AssignVariableOp_713"
  input: "^AssignVariableOp_714"
  input: "^AssignVariableOp_715"
  input: "^AssignVariableOp_716"
  input: "^AssignVariableOp_717"
  input: "^AssignVariableOp_718"
  input: "^AssignVariableOp_719"
  input: "^AssignVariableOp_72"
  input: "^AssignVariableOp_720"
  input: "^AssignVariableOp_721"
  input: "^AssignVariableOp_722"
  input: "^AssignVariableOp_723"
  input: "^AssignVariableOp_724"
  input: "^AssignVariableOp_725"
  input: "^AssignVariableOp_726"
  input: "^AssignVariableOp_727"
  input: "^AssignVariableOp_728"
  input: "^AssignVariableOp_729"
  input: "^AssignVariableOp_73"
  input: "^AssignVariableOp_730"
  input: "^AssignVariableOp_731"
  input: "^AssignVariableOp_732"
  input: "^AssignVariableOp_733"
  input: "^AssignVariableOp_734"
  input: "^AssignVariableOp_735"
  input: "^AssignVariableOp_736"
  input: "^AssignVariableOp_737"
  input: "^AssignVariableOp_738"
  input: "^AssignVariableOp_739"
  input: "^AssignVariableOp_74"
  input: "^AssignVariableOp_740"
  input: "^AssignVariableOp_741"
  input: "^AssignVariableOp_742"
  input: "^AssignVariableOp_743"
  input: "^AssignVariableOp_744"
  input: "^AssignVariableOp_745"
  input: "^AssignVariableOp_746"
  input: "^AssignVariableOp_747"
  input: "^AssignVariableOp_748"
  input: "^AssignVariableOp_749"
  input: "^AssignVariableOp_75"
  input: "^AssignVariableOp_750"
  input: "^AssignVariableOp_751"
  input: "^AssignVariableOp_752"
  input: "^AssignVariableOp_753"
  input: "^AssignVariableOp_754"
  input: "^AssignVariableOp_755"
  input: "^AssignVariableOp_756"
  input: "^AssignVariableOp_757"
  input: "^AssignVariableOp_758"
  input: "^AssignVariableOp_759"
  input: "^AssignVariableOp_76"
  input: "^AssignVariableOp_760"
  input: "^AssignVariableOp_761"
  input: "^AssignVariableOp_762"
  input: "^AssignVariableOp_763"
  input: "^AssignVariableOp_764"
  input: "^AssignVariableOp_765"
  input: "^AssignVariableOp_766"
  input: "^AssignVariableOp_767"
  input: "^AssignVariableOp_768"
  input: "^AssignVariableOp_769"
  input: "^AssignVariableOp_77"
  input: "^AssignVariableOp_770"
  input: "^AssignVariableOp_771"
  input: "^AssignVariableOp_772"
  input: "^AssignVariableOp_773"
  input: "^AssignVariableOp_774"
  input: "^AssignVariableOp_775"
  input: "^AssignVariableOp_776"
  input: "^AssignVariableOp_777"
  input: "^AssignVariableOp_778"
  input: "^AssignVariableOp_779"
  input: "^AssignVariableOp_78"
  input: "^AssignVariableOp_780"
  input: "^AssignVariableOp_781"
  input: "^AssignVariableOp_782"
  input: "^AssignVariableOp_783"
  input: "^AssignVariableOp_784"
  input: "^AssignVariableOp_785"
  input: "^AssignVariableOp_786"
  input: "^AssignVariableOp_787"
  input: "^AssignVariableOp_788"
  input: "^AssignVariableOp_789"
  input: "^AssignVariableOp_79"
  input: "^AssignVariableOp_790"
  input: "^AssignVariableOp_791"
  input: "^AssignVariableOp_792"
  input: "^AssignVariableOp_793"
  input: "^AssignVariableOp_794"
  input: "^AssignVariableOp_795"
  input: "^AssignVariableOp_796"
  input: "^AssignVariableOp_797"
  input: "^AssignVariableOp_798"
  input: "^AssignVariableOp_799"
  input: "^AssignVariableOp_8"
  input: "^AssignVariableOp_80"
  input: "^AssignVariableOp_800"
  input: "^AssignVariableOp_801"
  input: "^AssignVariableOp_802"
  input: "^AssignVariableOp_803"
  input: "^AssignVariableOp_804"
  input: "^AssignVariableOp_805"
  input: "^AssignVariableOp_806"
  input: "^AssignVariableOp_807"
  input: "^AssignVariableOp_808"
  input: "^AssignVariableOp_809"
  input: "^AssignVariableOp_81"
  input: "^AssignVariableOp_810"
  input: "^AssignVariableOp_811"
  input: "^AssignVariableOp_812"
  input: "^AssignVariableOp_813"
  input: "^AssignVariableOp_814"
  input: "^AssignVariableOp_815"
  input: "^AssignVariableOp_816"
  input: "^AssignVariableOp_817"
  input: "^AssignVariableOp_818"
  input: "^AssignVariableOp_819"
  input: "^AssignVariableOp_82"
  input: "^AssignVariableOp_820"
  input: "^AssignVariableOp_821"
  input: "^AssignVariableOp_822"
  input: "^AssignVariableOp_823"
  input: "^AssignVariableOp_824"
  input: "^AssignVariableOp_825"
  input: "^AssignVariableOp_826"
  input: "^AssignVariableOp_827"
  input: "^AssignVariableOp_828"
  input: "^AssignVariableOp_829"
  input: "^AssignVariableOp_83"
  input: "^AssignVariableOp_830"
  input: "^AssignVariableOp_831"
  input: "^AssignVariableOp_832"
  input: "^AssignVariableOp_833"
  input: "^AssignVariableOp_834"
  input: "^AssignVariableOp_835"
  input: "^AssignVariableOp_836"
  input: "^AssignVariableOp_837"
  input: "^AssignVariableOp_838"
  input: "^AssignVariableOp_839"
  input: "^AssignVariableOp_84"
  input: "^AssignVariableOp_840"
  input: "^AssignVariableOp_841"
  input: "^AssignVariableOp_842"
  input: "^AssignVariableOp_843"
  input: "^AssignVariableOp_844"
  input: "^AssignVariableOp_845"
  input: "^AssignVariableOp_846"
  input: "^AssignVariableOp_847"
  input: "^AssignVariableOp_848"
  input: "^AssignVariableOp_849"
  input: "^AssignVariableOp_85"
  input: "^AssignVariableOp_850"
  input: "^AssignVariableOp_851"
  input: "^AssignVariableOp_852"
  input: "^AssignVariableOp_853"
  input: "^AssignVariableOp_854"
  input: "^AssignVariableOp_855"
  input: "^AssignVariableOp_856"
  input: "^AssignVariableOp_857"
  input: "^AssignVariableOp_858"
  input: "^AssignVariableOp_859"
  input: "^AssignVariableOp_86"
  input: "^AssignVariableOp_860"
  input: "^AssignVariableOp_861"
  input: "^AssignVariableOp_862"
  input: "^AssignVariableOp_863"
  input: "^AssignVariableOp_864"
  input: "^AssignVariableOp_865"
  input: "^AssignVariableOp_866"
  input: "^AssignVariableOp_867"
  input: "^AssignVariableOp_868"
  input: "^AssignVariableOp_869"
  input: "^AssignVariableOp_87"
  input: "^AssignVariableOp_870"
  input: "^AssignVariableOp_871"
  input: "^AssignVariableOp_872"
  input: "^AssignVariableOp_873"
  input: "^AssignVariableOp_874"
  input: "^AssignVariableOp_875"
  input: "^AssignVariableOp_876"
  input: "^AssignVariableOp_877"
  input: "^AssignVariableOp_878"
  input: "^AssignVariableOp_879"
  input: "^AssignVariableOp_88"
  input: "^AssignVariableOp_880"
  input: "^AssignVariableOp_881"
  input: "^AssignVariableOp_882"
  input: "^AssignVariableOp_883"
  input: "^AssignVariableOp_884"
  input: "^AssignVariableOp_885"
  input: "^AssignVariableOp_886"
  input: "^AssignVariableOp_887"
  input: "^AssignVariableOp_888"
  input: "^AssignVariableOp_889"
  input: "^AssignVariableOp_89"
  input: "^AssignVariableOp_890"
  input: "^AssignVariableOp_891"
  input: "^AssignVariableOp_892"
  input: "^AssignVariableOp_893"
  input: "^AssignVariableOp_894"
  input: "^AssignVariableOp_895"
  input: "^AssignVariableOp_896"
  input: "^AssignVariableOp_897"
  input: "^AssignVariableOp_898"
  input: "^AssignVariableOp_899"
  input: "^AssignVariableOp_9"
  input: "^AssignVariableOp_90"
  input: "^AssignVariableOp_900"
  input: "^AssignVariableOp_901"
  input: "^AssignVariableOp_902"
  input: "^AssignVariableOp_903"
  input: "^AssignVariableOp_904"
  input: "^AssignVariableOp_905"
  input: "^AssignVariableOp_906"
  input: "^AssignVariableOp_907"
  input: "^AssignVariableOp_908"
  input: "^AssignVariableOp_909"
  input: "^AssignVariableOp_91"
  input: "^AssignVariableOp_910"
  input: "^AssignVariableOp_911"
  input: "^AssignVariableOp_912"
  input: "^AssignVariableOp_913"
  input: "^AssignVariableOp_914"
  input: "^AssignVariableOp_915"
  input: "^AssignVariableOp_916"
  input: "^AssignVariableOp_917"
  input: "^AssignVariableOp_918"
  input: "^AssignVariableOp_919"
  input: "^AssignVariableOp_92"
  input: "^AssignVariableOp_920"
  input: "^AssignVariableOp_921"
  input: "^AssignVariableOp_922"
  input: "^AssignVariableOp_923"
  input: "^AssignVariableOp_924"
  input: "^AssignVariableOp_925"
  input: "^AssignVariableOp_926"
  input: "^AssignVariableOp_927"
  input: "^AssignVariableOp_928"
  input: "^AssignVariableOp_929"
  input: "^AssignVariableOp_93"
  input: "^AssignVariableOp_930"
  input: "^AssignVariableOp_931"
  input: "^AssignVariableOp_932"
  input: "^AssignVariableOp_933"
  input: "^AssignVariableOp_934"
  input: "^AssignVariableOp_935"
  input: "^AssignVariableOp_936"
  input: "^AssignVariableOp_937"
  input: "^AssignVariableOp_938"
  input: "^AssignVariableOp_939"
  input: "^AssignVariableOp_94"
  input: "^AssignVariableOp_940"
  input: "^AssignVariableOp_941"
  input: "^AssignVariableOp_942"
  input: "^AssignVariableOp_943"
  input: "^AssignVariableOp_944"
  input: "^AssignVariableOp_945"
  input: "^AssignVariableOp_946"
  input: "^AssignVariableOp_947"
  input: "^AssignVariableOp_948"
  input: "^AssignVariableOp_949"
  input: "^AssignVariableOp_95"
  input: "^AssignVariableOp_950"
  input: "^AssignVariableOp_951"
  input: "^AssignVariableOp_952"
  input: "^AssignVariableOp_953"
  input: "^AssignVariableOp_954"
  input: "^AssignVariableOp_955"
  input: "^AssignVariableOp_956"
  input: "^AssignVariableOp_957"
  input: "^AssignVariableOp_958"
  input: "^AssignVariableOp_959"
  input: "^AssignVariableOp_96"
  input: "^AssignVariableOp_960"
  input: "^AssignVariableOp_961"
  input: "^AssignVariableOp_962"
  input: "^AssignVariableOp_963"
  input: "^AssignVariableOp_964"
  input: "^AssignVariableOp_965"
  input: "^AssignVariableOp_966"
  input: "^AssignVariableOp_967"
  input: "^AssignVariableOp_968"
  input: "^AssignVariableOp_969"
  input: "^AssignVariableOp_97"
  input: "^AssignVariableOp_970"
  input: "^AssignVariableOp_971"
  input: "^AssignVariableOp_972"
  input: "^AssignVariableOp_973"
  input: "^AssignVariableOp_974"
  input: "^AssignVariableOp_975"
  input: "^AssignVariableOp_976"
  input: "^AssignVariableOp_977"
  input: "^AssignVariableOp_978"
  input: "^AssignVariableOp_979"
  input: "^AssignVariableOp_98"
  input: "^AssignVariableOp_980"
  input: "^AssignVariableOp_981"
  input: "^AssignVariableOp_982"
  input: "^AssignVariableOp_983"
  input: "^AssignVariableOp_984"
  input: "^AssignVariableOp_985"
  input: "^AssignVariableOp_986"
  input: "^AssignVariableOp_987"
  input: "^AssignVariableOp_988"
  input: "^AssignVariableOp_989"
  input: "^AssignVariableOp_99"
  input: "^AssignVariableOp_990"
  input: "^AssignVariableOp_991"
  input: "^AssignVariableOp_992"
  input: "^AssignVariableOp_993"
  input: "^AssignVariableOp_994"
  input: "^AssignVariableOp_995"
  input: "^AssignVariableOp_996"
  input: "^AssignVariableOp_997"
  input: "^AssignVariableOp_998"
  input: "^AssignVariableOp_999"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "ReadVariableOp_1759"
  op: "ReadVariableOp"
  input: "PolynomialDecay/Cast_2/ReadVariableOp/Enter"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "add_1369/y"
  op: "Const"
  input: "^Identity"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 1
      }
    }
  }
}
node {
  name: "add_1369"
  op: "AddV2"
  input: "ReadVariableOp_1759"
  input: "add_1369/y"
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "AssignVariableOp_1206"
  op: "AssignVariableOp"
  input: "PolynomialDecay/Cast_2/ReadVariableOp/Enter"
  input: "add_1369"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "ReadVariableOp_1760"
  op: "ReadVariableOp"
  input: "PolynomialDecay/Cast_2/ReadVariableOp/Enter"
  input: "^AssignVariableOp_1206"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "group_deps_1"
  op: "NoOp"
  input: "^AssignVariableOp_1206"
  input: "^group_deps"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Reshape_16/shape"
  op: "Const"
  input: "^group_deps_1"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "Reshape_16"
  op: "Reshape"
  input: "add_11"
  input: "Reshape_16/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "OutfeedEnqueueTuple"
  op: "OutfeedEnqueueTuple"
  input: "Reshape_16"
  device: "/device:TPU_REPLICATED_CORE:0"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_FLOAT
      }
    }
  }
}
node {
  name: "Identity_2"
  op: "Identity"
  input: "add_11"
  input: "^OutfeedEnqueueTuple"
  input: "^group_deps_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "NextIteration"
  op: "NextIteration"
  input: "add"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "NextIteration_1"
  op: "NextIteration"
  input: "Identity_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Exit"
  op: "Exit"
  input: "Switch"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Exit_1"
  op: "Exit"
  input: "Switch_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "NoOp"
  op: "NoOp"
  input: "^cluster/pivot"
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "Identity_3"
  op: "Identity"
  input: "Exit_1"
  device: "/device:TPU_REPLICATED_CORE:0"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_output_identity"
    value {
      b: true
    }
  }
  attr {
    key: "_tpu_replicate"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "TPUCompilationResult"
  op: "TPUCompilationResult"
  input: "^TPUReplicateMetadata"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "_tpu_compilation_status"
    value {
      s: "cluster"
    }
  }
}
node {
  name: "output0"
  op: "TPUReplicatedOutput"
  input: "Identity_3"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
        shape {
        }
        shape {
        }
        shape {
        }
        shape {
        }
        shape {
        }
        shape {
        }
      }
    }
  }
  attr {
    key: "num_replicas"
    value {
      i: 8
    }
  }
}
node {
  name: "output_0_shard_0"
  op: "Identity"
  input: "output0"
  input: "^NoOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "output_0_shard_1"
  op: "Identity"
  input: "output0:1"
  input: "^NoOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "output_0_shard_2"
  op: "Identity"
  input: "output0:2"
  input: "^NoOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "output_0_shard_3"
  op: "Identity"
  input: "output0:3"
  input: "^NoOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "output_0_shard_4"
  op: "Identity"
  input: "output0:4"
  input: "^NoOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "output_0_shard_5"
  op: "Identity"
  input: "output0:5"
  input: "^NoOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "output_0_shard_6"
  op: "Identity"
  input: "output0:6"
  input: "^NoOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "output_0_shard_7"
  op: "Identity"
  input: "output0:7"
  input: "^NoOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "OutfeedDequeueTuple"
  op: "OutfeedDequeueTuple"
  device: "/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 0
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_FLOAT
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "OutfeedDequeueTuple_1"
  op: "OutfeedDequeueTuple"
  device: "/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 1
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_FLOAT
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "OutfeedDequeueTuple_2"
  op: "OutfeedDequeueTuple"
  device: "/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 2
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_FLOAT
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "OutfeedDequeueTuple_3"
  op: "OutfeedDequeueTuple"
  device: "/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 3
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_FLOAT
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "OutfeedDequeueTuple_4"
  op: "OutfeedDequeueTuple"
  device: "/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 4
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_FLOAT
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "OutfeedDequeueTuple_5"
  op: "OutfeedDequeueTuple"
  device: "/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 5
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_FLOAT
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "OutfeedDequeueTuple_6"
  op: "OutfeedDequeueTuple"
  device: "/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 6
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_FLOAT
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "OutfeedDequeueTuple_7"
  op: "OutfeedDequeueTuple"
  device: "/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "device_ordinal"
    value {
      i: 7
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_FLOAT
      }
    }
  }
  attr {
    key: "shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "concat_3/axis"
  op: "Const"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "concat_3"
  op: "ConcatV2"
  input: "OutfeedDequeueTuple"
  input: "OutfeedDequeueTuple_1"
  input: "OutfeedDequeueTuple_2"
  input: "OutfeedDequeueTuple_3"
  input: "OutfeedDequeueTuple_4"
  input: "OutfeedDequeueTuple_5"
  input: "OutfeedDequeueTuple_6"
  input: "OutfeedDequeueTuple_7"
  input: "concat_3/axis"
  device: "/replica:0/task:0/device:CPU:0"
  attr {
    key: "N"
    value {
      i: 8
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 8
          }
        }
      }
    }
  }
}
node {
  name: "group_deps_2"
  op: "NoOp"
  input: "^OutfeedDequeueTuple"
  input: "^OutfeedDequeueTuple_1"
  input: "^OutfeedDequeueTuple_2"
  input: "^OutfeedDequeueTuple_3"
  input: "^OutfeedDequeueTuple_4"
  input: "^OutfeedDequeueTuple_5"
  input: "^OutfeedDequeueTuple_6"
  input: "^OutfeedDequeueTuple_7"
  device: "/task:0/device:CPU:0"
}
node {
  name: "Identity_4/ReadVariableOp"
  op: "ReadVariableOp"
  input: "global_step"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "Identity_4"
  op: "Identity"
  input: "Identity_4/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "Identity_5"
  op: "Identity"
  input: "output_0_shard_0"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "loss/tags"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "loss"
      }
    }
  }
}
node {
  name: "loss"
  op: "ScalarSummary"
  input: "loss/tags"
  input: "output_0_shard_0"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/word_embeddings"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_6"
  op: "Identity"
  input: "Read/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics"
  op: "CheckNumerics"
  input: "Identity_6"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/embeddings/word_embeddings:0 is NaN"
    }
  }
}
node {
  name: "Read_1/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/token_type_embeddings"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_7"
  op: "Identity"
  input: "Read_1/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_1"
  op: "CheckNumerics"
  input: "Identity_7"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/embeddings/token_type_embeddings:0 is NaN"
    }
  }
}
node {
  name: "Read_2/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings/position_embeddings"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_8"
  op: "Identity"
  input: "Read_2/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_2"
  op: "CheckNumerics"
  input: "Identity_8"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/embeddings/position_embeddings:0 is NaN"
    }
  }
}
node {
  name: "Read_3/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_9"
  op: "Identity"
  input: "Read_3/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_3"
  op: "CheckNumerics"
  input: "Identity_9"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/embeddings_1/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_4/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_10"
  op: "Identity"
  input: "Read_4/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_4"
  op: "CheckNumerics"
  input: "Identity_10"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/embeddings_1/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_5/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_11"
  op: "Identity"
  input: "Read_5/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_5"
  op: "CheckNumerics"
  input: "Identity_11"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/embeddings_project/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_6/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/embeddings_project/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_12"
  op: "Identity"
  input: "Read_6/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_6"
  op: "CheckNumerics"
  input: "Identity_12"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/embeddings_project/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_7/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_13"
  op: "Identity"
  input: "Read_7/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_7"
  op: "CheckNumerics"
  input: "Identity_13"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_8/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_14"
  op: "Identity"
  input: "Read_8/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_8"
  op: "CheckNumerics"
  input: "Identity_14"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_9/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_15"
  op: "Identity"
  input: "Read_9/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_9"
  op: "CheckNumerics"
  input: "Identity_15"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_10/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_16"
  op: "Identity"
  input: "Read_10/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_10"
  op: "CheckNumerics"
  input: "Identity_16"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_11/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_17"
  op: "Identity"
  input: "Read_11/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_11"
  op: "CheckNumerics"
  input: "Identity_17"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_12/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_18"
  op: "Identity"
  input: "Read_12/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_12"
  op: "CheckNumerics"
  input: "Identity_18"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_13/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_19"
  op: "Identity"
  input: "Read_13/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_13"
  op: "CheckNumerics"
  input: "Identity_19"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_14/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_20"
  op: "Identity"
  input: "Read_14/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_14"
  op: "CheckNumerics"
  input: "Identity_20"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_15/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_21"
  op: "Identity"
  input: "Read_15/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_15"
  op: "CheckNumerics"
  input: "Identity_21"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_16/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_22"
  op: "Identity"
  input: "Read_16/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_16"
  op: "CheckNumerics"
  input: "Identity_22"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_17/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_23"
  op: "Identity"
  input: "Read_17/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_17"
  op: "CheckNumerics"
  input: "Identity_23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_18/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_24"
  op: "Identity"
  input: "Read_18/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_18"
  op: "CheckNumerics"
  input: "Identity_24"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_19/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_25"
  op: "Identity"
  input: "Read_19/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_19"
  op: "CheckNumerics"
  input: "Identity_25"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_20/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_26"
  op: "Identity"
  input: "Read_20/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_20"
  op: "CheckNumerics"
  input: "Identity_26"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_21/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_27"
  op: "Identity"
  input: "Read_21/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_21"
  op: "CheckNumerics"
  input: "Identity_27"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_22/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_28"
  op: "Identity"
  input: "Read_22/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_22"
  op: "CheckNumerics"
  input: "Identity_28"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_0/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_23/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_29"
  op: "Identity"
  input: "Read_23/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_23"
  op: "CheckNumerics"
  input: "Identity_29"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_24/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_30"
  op: "Identity"
  input: "Read_24/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_24"
  op: "CheckNumerics"
  input: "Identity_30"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_25/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_31"
  op: "Identity"
  input: "Read_25/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_25"
  op: "CheckNumerics"
  input: "Identity_31"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_26/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_32"
  op: "Identity"
  input: "Read_26/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_26"
  op: "CheckNumerics"
  input: "Identity_32"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_27/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_33"
  op: "Identity"
  input: "Read_27/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_27"
  op: "CheckNumerics"
  input: "Identity_33"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_28/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_34"
  op: "Identity"
  input: "Read_28/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_28"
  op: "CheckNumerics"
  input: "Identity_34"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_29/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_35"
  op: "Identity"
  input: "Read_29/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_29"
  op: "CheckNumerics"
  input: "Identity_35"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_30/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_36"
  op: "Identity"
  input: "Read_30/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_30"
  op: "CheckNumerics"
  input: "Identity_36"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_31/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_37"
  op: "Identity"
  input: "Read_31/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_31"
  op: "CheckNumerics"
  input: "Identity_37"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_32/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_38"
  op: "Identity"
  input: "Read_32/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_32"
  op: "CheckNumerics"
  input: "Identity_38"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_33/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_39"
  op: "Identity"
  input: "Read_33/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_33"
  op: "CheckNumerics"
  input: "Identity_39"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_34/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_40"
  op: "Identity"
  input: "Read_34/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_34"
  op: "CheckNumerics"
  input: "Identity_40"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_35/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_41"
  op: "Identity"
  input: "Read_35/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_35"
  op: "CheckNumerics"
  input: "Identity_41"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_36/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_42"
  op: "Identity"
  input: "Read_36/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_36"
  op: "CheckNumerics"
  input: "Identity_42"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_37/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_43"
  op: "Identity"
  input: "Read_37/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_37"
  op: "CheckNumerics"
  input: "Identity_43"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_38/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_44"
  op: "Identity"
  input: "Read_38/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_38"
  op: "CheckNumerics"
  input: "Identity_44"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_1/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_39/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_45"
  op: "Identity"
  input: "Read_39/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_39"
  op: "CheckNumerics"
  input: "Identity_45"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_40/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_46"
  op: "Identity"
  input: "Read_40/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_40"
  op: "CheckNumerics"
  input: "Identity_46"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_41/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_47"
  op: "Identity"
  input: "Read_41/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_41"
  op: "CheckNumerics"
  input: "Identity_47"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_42/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_48"
  op: "Identity"
  input: "Read_42/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_42"
  op: "CheckNumerics"
  input: "Identity_48"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_43/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_49"
  op: "Identity"
  input: "Read_43/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_43"
  op: "CheckNumerics"
  input: "Identity_49"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_44/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_50"
  op: "Identity"
  input: "Read_44/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_44"
  op: "CheckNumerics"
  input: "Identity_50"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_45/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_51"
  op: "Identity"
  input: "Read_45/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_45"
  op: "CheckNumerics"
  input: "Identity_51"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_46/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_52"
  op: "Identity"
  input: "Read_46/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_46"
  op: "CheckNumerics"
  input: "Identity_52"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_47/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_53"
  op: "Identity"
  input: "Read_47/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_47"
  op: "CheckNumerics"
  input: "Identity_53"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_48/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_54"
  op: "Identity"
  input: "Read_48/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_48"
  op: "CheckNumerics"
  input: "Identity_54"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_49/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_55"
  op: "Identity"
  input: "Read_49/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_49"
  op: "CheckNumerics"
  input: "Identity_55"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_50/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_56"
  op: "Identity"
  input: "Read_50/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_50"
  op: "CheckNumerics"
  input: "Identity_56"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_51/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_57"
  op: "Identity"
  input: "Read_51/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_51"
  op: "CheckNumerics"
  input: "Identity_57"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_52/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_58"
  op: "Identity"
  input: "Read_52/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_52"
  op: "CheckNumerics"
  input: "Identity_58"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_53/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_59"
  op: "Identity"
  input: "Read_53/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_53"
  op: "CheckNumerics"
  input: "Identity_59"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_54/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_60"
  op: "Identity"
  input: "Read_54/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_54"
  op: "CheckNumerics"
  input: "Identity_60"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_2/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_55/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_61"
  op: "Identity"
  input: "Read_55/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_55"
  op: "CheckNumerics"
  input: "Identity_61"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_56/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_62"
  op: "Identity"
  input: "Read_56/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_56"
  op: "CheckNumerics"
  input: "Identity_62"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_57/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_63"
  op: "Identity"
  input: "Read_57/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_57"
  op: "CheckNumerics"
  input: "Identity_63"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_58/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_64"
  op: "Identity"
  input: "Read_58/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_58"
  op: "CheckNumerics"
  input: "Identity_64"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_59/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_65"
  op: "Identity"
  input: "Read_59/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_59"
  op: "CheckNumerics"
  input: "Identity_65"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_60/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_66"
  op: "Identity"
  input: "Read_60/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_60"
  op: "CheckNumerics"
  input: "Identity_66"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_61/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_67"
  op: "Identity"
  input: "Read_61/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_61"
  op: "CheckNumerics"
  input: "Identity_67"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_62/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_68"
  op: "Identity"
  input: "Read_62/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_62"
  op: "CheckNumerics"
  input: "Identity_68"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_63/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_69"
  op: "Identity"
  input: "Read_63/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_63"
  op: "CheckNumerics"
  input: "Identity_69"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_64/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_70"
  op: "Identity"
  input: "Read_64/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_64"
  op: "CheckNumerics"
  input: "Identity_70"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_65/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_71"
  op: "Identity"
  input: "Read_65/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_65"
  op: "CheckNumerics"
  input: "Identity_71"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_66/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_72"
  op: "Identity"
  input: "Read_66/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_66"
  op: "CheckNumerics"
  input: "Identity_72"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_67/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_73"
  op: "Identity"
  input: "Read_67/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_67"
  op: "CheckNumerics"
  input: "Identity_73"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_68/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_74"
  op: "Identity"
  input: "Read_68/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_68"
  op: "CheckNumerics"
  input: "Identity_74"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_69/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_75"
  op: "Identity"
  input: "Read_69/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_69"
  op: "CheckNumerics"
  input: "Identity_75"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_70/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_76"
  op: "Identity"
  input: "Read_70/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_70"
  op: "CheckNumerics"
  input: "Identity_76"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_3/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_71/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_77"
  op: "Identity"
  input: "Read_71/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_71"
  op: "CheckNumerics"
  input: "Identity_77"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_72/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_78"
  op: "Identity"
  input: "Read_72/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_72"
  op: "CheckNumerics"
  input: "Identity_78"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_73/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_79"
  op: "Identity"
  input: "Read_73/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_73"
  op: "CheckNumerics"
  input: "Identity_79"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_74/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_80"
  op: "Identity"
  input: "Read_74/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_74"
  op: "CheckNumerics"
  input: "Identity_80"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_75/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_81"
  op: "Identity"
  input: "Read_75/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_75"
  op: "CheckNumerics"
  input: "Identity_81"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_76/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_82"
  op: "Identity"
  input: "Read_76/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_76"
  op: "CheckNumerics"
  input: "Identity_82"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_77/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_83"
  op: "Identity"
  input: "Read_77/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_77"
  op: "CheckNumerics"
  input: "Identity_83"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_78/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_84"
  op: "Identity"
  input: "Read_78/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_78"
  op: "CheckNumerics"
  input: "Identity_84"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_79/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_85"
  op: "Identity"
  input: "Read_79/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_79"
  op: "CheckNumerics"
  input: "Identity_85"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_80/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_86"
  op: "Identity"
  input: "Read_80/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_80"
  op: "CheckNumerics"
  input: "Identity_86"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_81/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_87"
  op: "Identity"
  input: "Read_81/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_81"
  op: "CheckNumerics"
  input: "Identity_87"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_82/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_88"
  op: "Identity"
  input: "Read_82/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_82"
  op: "CheckNumerics"
  input: "Identity_88"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_83/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_89"
  op: "Identity"
  input: "Read_83/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_83"
  op: "CheckNumerics"
  input: "Identity_89"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_84/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_90"
  op: "Identity"
  input: "Read_84/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_84"
  op: "CheckNumerics"
  input: "Identity_90"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_85/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_91"
  op: "Identity"
  input: "Read_85/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_85"
  op: "CheckNumerics"
  input: "Identity_91"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_86/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_92"
  op: "Identity"
  input: "Read_86/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_86"
  op: "CheckNumerics"
  input: "Identity_92"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_4/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_87/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_93"
  op: "Identity"
  input: "Read_87/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_87"
  op: "CheckNumerics"
  input: "Identity_93"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_88/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_94"
  op: "Identity"
  input: "Read_88/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_88"
  op: "CheckNumerics"
  input: "Identity_94"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_89/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_95"
  op: "Identity"
  input: "Read_89/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_89"
  op: "CheckNumerics"
  input: "Identity_95"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_90/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_96"
  op: "Identity"
  input: "Read_90/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_90"
  op: "CheckNumerics"
  input: "Identity_96"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_91/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_97"
  op: "Identity"
  input: "Read_91/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_91"
  op: "CheckNumerics"
  input: "Identity_97"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_92/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_98"
  op: "Identity"
  input: "Read_92/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_92"
  op: "CheckNumerics"
  input: "Identity_98"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_93/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_99"
  op: "Identity"
  input: "Read_93/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_93"
  op: "CheckNumerics"
  input: "Identity_99"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_94/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_100"
  op: "Identity"
  input: "Read_94/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_94"
  op: "CheckNumerics"
  input: "Identity_100"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_95/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_101"
  op: "Identity"
  input: "Read_95/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_95"
  op: "CheckNumerics"
  input: "Identity_101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_96/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_102"
  op: "Identity"
  input: "Read_96/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_96"
  op: "CheckNumerics"
  input: "Identity_102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_97/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_103"
  op: "Identity"
  input: "Read_97/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_97"
  op: "CheckNumerics"
  input: "Identity_103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_98/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_104"
  op: "Identity"
  input: "Read_98/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_98"
  op: "CheckNumerics"
  input: "Identity_104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_99/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_105"
  op: "Identity"
  input: "Read_99/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_99"
  op: "CheckNumerics"
  input: "Identity_105"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_100/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_106"
  op: "Identity"
  input: "Read_100/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_100"
  op: "CheckNumerics"
  input: "Identity_106"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_101/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_107"
  op: "Identity"
  input: "Read_101/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_101"
  op: "CheckNumerics"
  input: "Identity_107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_102/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_108"
  op: "Identity"
  input: "Read_102/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_102"
  op: "CheckNumerics"
  input: "Identity_108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_5/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_103/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_109"
  op: "Identity"
  input: "Read_103/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_103"
  op: "CheckNumerics"
  input: "Identity_109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_104/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_110"
  op: "Identity"
  input: "Read_104/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_104"
  op: "CheckNumerics"
  input: "Identity_110"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_105/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_111"
  op: "Identity"
  input: "Read_105/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_105"
  op: "CheckNumerics"
  input: "Identity_111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_106/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_112"
  op: "Identity"
  input: "Read_106/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_106"
  op: "CheckNumerics"
  input: "Identity_112"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_107/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_113"
  op: "Identity"
  input: "Read_107/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_107"
  op: "CheckNumerics"
  input: "Identity_113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_108/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_114"
  op: "Identity"
  input: "Read_108/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_108"
  op: "CheckNumerics"
  input: "Identity_114"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_109/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_115"
  op: "Identity"
  input: "Read_109/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_109"
  op: "CheckNumerics"
  input: "Identity_115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_110/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_116"
  op: "Identity"
  input: "Read_110/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_110"
  op: "CheckNumerics"
  input: "Identity_116"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_111/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_117"
  op: "Identity"
  input: "Read_111/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_111"
  op: "CheckNumerics"
  input: "Identity_117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_112/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_118"
  op: "Identity"
  input: "Read_112/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_112"
  op: "CheckNumerics"
  input: "Identity_118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_113/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_119"
  op: "Identity"
  input: "Read_113/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_113"
  op: "CheckNumerics"
  input: "Identity_119"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_114/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_120"
  op: "Identity"
  input: "Read_114/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_114"
  op: "CheckNumerics"
  input: "Identity_120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_115/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_121"
  op: "Identity"
  input: "Read_115/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_115"
  op: "CheckNumerics"
  input: "Identity_121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_116/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_122"
  op: "Identity"
  input: "Read_116/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_116"
  op: "CheckNumerics"
  input: "Identity_122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_117/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_123"
  op: "Identity"
  input: "Read_117/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_117"
  op: "CheckNumerics"
  input: "Identity_123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_118/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_124"
  op: "Identity"
  input: "Read_118/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_118"
  op: "CheckNumerics"
  input: "Identity_124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_6/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_119/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_125"
  op: "Identity"
  input: "Read_119/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_119"
  op: "CheckNumerics"
  input: "Identity_125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_120/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_126"
  op: "Identity"
  input: "Read_120/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_120"
  op: "CheckNumerics"
  input: "Identity_126"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_121/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_127"
  op: "Identity"
  input: "Read_121/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_121"
  op: "CheckNumerics"
  input: "Identity_127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_122/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_128"
  op: "Identity"
  input: "Read_122/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_122"
  op: "CheckNumerics"
  input: "Identity_128"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_123/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_129"
  op: "Identity"
  input: "Read_123/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_123"
  op: "CheckNumerics"
  input: "Identity_129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_124/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_130"
  op: "Identity"
  input: "Read_124/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_124"
  op: "CheckNumerics"
  input: "Identity_130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_125/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_131"
  op: "Identity"
  input: "Read_125/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_125"
  op: "CheckNumerics"
  input: "Identity_131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_126/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_132"
  op: "Identity"
  input: "Read_126/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_126"
  op: "CheckNumerics"
  input: "Identity_132"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_127/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_133"
  op: "Identity"
  input: "Read_127/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_127"
  op: "CheckNumerics"
  input: "Identity_133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_128/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_134"
  op: "Identity"
  input: "Read_128/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_128"
  op: "CheckNumerics"
  input: "Identity_134"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_129/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_135"
  op: "Identity"
  input: "Read_129/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_129"
  op: "CheckNumerics"
  input: "Identity_135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_130/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_136"
  op: "Identity"
  input: "Read_130/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_130"
  op: "CheckNumerics"
  input: "Identity_136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_131/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_137"
  op: "Identity"
  input: "Read_131/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_131"
  op: "CheckNumerics"
  input: "Identity_137"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_132/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_138"
  op: "Identity"
  input: "Read_132/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_132"
  op: "CheckNumerics"
  input: "Identity_138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_133/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_139"
  op: "Identity"
  input: "Read_133/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_133"
  op: "CheckNumerics"
  input: "Identity_139"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_134/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_140"
  op: "Identity"
  input: "Read_134/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_134"
  op: "CheckNumerics"
  input: "Identity_140"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_7/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_135/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_141"
  op: "Identity"
  input: "Read_135/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_135"
  op: "CheckNumerics"
  input: "Identity_141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_136/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_142"
  op: "Identity"
  input: "Read_136/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_136"
  op: "CheckNumerics"
  input: "Identity_142"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_137/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_143"
  op: "Identity"
  input: "Read_137/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_137"
  op: "CheckNumerics"
  input: "Identity_143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_138/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_144"
  op: "Identity"
  input: "Read_138/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_138"
  op: "CheckNumerics"
  input: "Identity_144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_139/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_145"
  op: "Identity"
  input: "Read_139/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_139"
  op: "CheckNumerics"
  input: "Identity_145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_140/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_146"
  op: "Identity"
  input: "Read_140/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_140"
  op: "CheckNumerics"
  input: "Identity_146"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_141/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_147"
  op: "Identity"
  input: "Read_141/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_141"
  op: "CheckNumerics"
  input: "Identity_147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_142/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_148"
  op: "Identity"
  input: "Read_142/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_142"
  op: "CheckNumerics"
  input: "Identity_148"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_143/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_149"
  op: "Identity"
  input: "Read_143/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_143"
  op: "CheckNumerics"
  input: "Identity_149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_144/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_150"
  op: "Identity"
  input: "Read_144/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_144"
  op: "CheckNumerics"
  input: "Identity_150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_145/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_151"
  op: "Identity"
  input: "Read_145/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_145"
  op: "CheckNumerics"
  input: "Identity_151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_146/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_152"
  op: "Identity"
  input: "Read_146/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_146"
  op: "CheckNumerics"
  input: "Identity_152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_147/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_153"
  op: "Identity"
  input: "Read_147/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_147"
  op: "CheckNumerics"
  input: "Identity_153"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_148/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_154"
  op: "Identity"
  input: "Read_148/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_148"
  op: "CheckNumerics"
  input: "Identity_154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_149/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_155"
  op: "Identity"
  input: "Read_149/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_149"
  op: "CheckNumerics"
  input: "Identity_155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_150/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_156"
  op: "Identity"
  input: "Read_150/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_150"
  op: "CheckNumerics"
  input: "Identity_156"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_8/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_151/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_157"
  op: "Identity"
  input: "Read_151/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_151"
  op: "CheckNumerics"
  input: "Identity_157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_152/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_158"
  op: "Identity"
  input: "Read_152/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_152"
  op: "CheckNumerics"
  input: "Identity_158"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_153/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_159"
  op: "Identity"
  input: "Read_153/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_153"
  op: "CheckNumerics"
  input: "Identity_159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_154/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_160"
  op: "Identity"
  input: "Read_154/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_154"
  op: "CheckNumerics"
  input: "Identity_160"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_155/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_161"
  op: "Identity"
  input: "Read_155/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_155"
  op: "CheckNumerics"
  input: "Identity_161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_156/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_162"
  op: "Identity"
  input: "Read_156/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_156"
  op: "CheckNumerics"
  input: "Identity_162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_157/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_163"
  op: "Identity"
  input: "Read_157/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_157"
  op: "CheckNumerics"
  input: "Identity_163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_158/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_164"
  op: "Identity"
  input: "Read_158/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_158"
  op: "CheckNumerics"
  input: "Identity_164"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_159/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_165"
  op: "Identity"
  input: "Read_159/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_159"
  op: "CheckNumerics"
  input: "Identity_165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_160/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_166"
  op: "Identity"
  input: "Read_160/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_160"
  op: "CheckNumerics"
  input: "Identity_166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_161/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_167"
  op: "Identity"
  input: "Read_161/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_161"
  op: "CheckNumerics"
  input: "Identity_167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_162/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_168"
  op: "Identity"
  input: "Read_162/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_162"
  op: "CheckNumerics"
  input: "Identity_168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_163/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_169"
  op: "Identity"
  input: "Read_163/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_163"
  op: "CheckNumerics"
  input: "Identity_169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_164/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_170"
  op: "Identity"
  input: "Read_164/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_164"
  op: "CheckNumerics"
  input: "Identity_170"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_165/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_171"
  op: "Identity"
  input: "Read_165/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_165"
  op: "CheckNumerics"
  input: "Identity_171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_166/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_172"
  op: "Identity"
  input: "Read_166/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_166"
  op: "CheckNumerics"
  input: "Identity_172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_9/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_167/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_173"
  op: "Identity"
  input: "Read_167/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_167"
  op: "CheckNumerics"
  input: "Identity_173"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_168/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_174"
  op: "Identity"
  input: "Read_168/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_168"
  op: "CheckNumerics"
  input: "Identity_174"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_169/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_175"
  op: "Identity"
  input: "Read_169/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_169"
  op: "CheckNumerics"
  input: "Identity_175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_170/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_176"
  op: "Identity"
  input: "Read_170/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_170"
  op: "CheckNumerics"
  input: "Identity_176"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_171/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_177"
  op: "Identity"
  input: "Read_171/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_171"
  op: "CheckNumerics"
  input: "Identity_177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_172/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_178"
  op: "Identity"
  input: "Read_172/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_172"
  op: "CheckNumerics"
  input: "Identity_178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_173/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_179"
  op: "Identity"
  input: "Read_173/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_173"
  op: "CheckNumerics"
  input: "Identity_179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_174/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_180"
  op: "Identity"
  input: "Read_174/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_174"
  op: "CheckNumerics"
  input: "Identity_180"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_175/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_181"
  op: "Identity"
  input: "Read_175/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_175"
  op: "CheckNumerics"
  input: "Identity_181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_176/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_182"
  op: "Identity"
  input: "Read_176/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_176"
  op: "CheckNumerics"
  input: "Identity_182"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_177/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_183"
  op: "Identity"
  input: "Read_177/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_177"
  op: "CheckNumerics"
  input: "Identity_183"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_178/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_184"
  op: "Identity"
  input: "Read_178/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_178"
  op: "CheckNumerics"
  input: "Identity_184"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_179/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_185"
  op: "Identity"
  input: "Read_179/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_179"
  op: "CheckNumerics"
  input: "Identity_185"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_180/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_186"
  op: "Identity"
  input: "Read_180/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_180"
  op: "CheckNumerics"
  input: "Identity_186"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_181/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_187"
  op: "Identity"
  input: "Read_181/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_181"
  op: "CheckNumerics"
  input: "Identity_187"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_182/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_188"
  op: "Identity"
  input: "Read_182/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_182"
  op: "CheckNumerics"
  input: "Identity_188"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_10/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_183/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_189"
  op: "Identity"
  input: "Read_183/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_183"
  op: "CheckNumerics"
  input: "Identity_189"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_184/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_190"
  op: "Identity"
  input: "Read_184/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_184"
  op: "CheckNumerics"
  input: "Identity_190"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_185/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_191"
  op: "Identity"
  input: "Read_185/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_185"
  op: "CheckNumerics"
  input: "Identity_191"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_186/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_192"
  op: "Identity"
  input: "Read_186/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_186"
  op: "CheckNumerics"
  input: "Identity_192"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_187/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_193"
  op: "Identity"
  input: "Read_187/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_187"
  op: "CheckNumerics"
  input: "Identity_193"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_188/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_194"
  op: "Identity"
  input: "Read_188/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_188"
  op: "CheckNumerics"
  input: "Identity_194"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_189/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_195"
  op: "Identity"
  input: "Read_189/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_189"
  op: "CheckNumerics"
  input: "Identity_195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_190/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_196"
  op: "Identity"
  input: "Read_190/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_190"
  op: "CheckNumerics"
  input: "Identity_196"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_191/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_197"
  op: "Identity"
  input: "Read_191/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_191"
  op: "CheckNumerics"
  input: "Identity_197"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_192/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_198"
  op: "Identity"
  input: "Read_192/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_192"
  op: "CheckNumerics"
  input: "Identity_198"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_193/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_199"
  op: "Identity"
  input: "Read_193/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_193"
  op: "CheckNumerics"
  input: "Identity_199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_194/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_200"
  op: "Identity"
  input: "Read_194/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_194"
  op: "CheckNumerics"
  input: "Identity_200"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_195/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_201"
  op: "Identity"
  input: "Read_195/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_195"
  op: "CheckNumerics"
  input: "Identity_201"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1024
          }
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_196/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_202"
  op: "Identity"
  input: "Read_196/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_196"
  op: "CheckNumerics"
  input: "Identity_202"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_197/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_203"
  op: "Identity"
  input: "Read_197/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_197"
  op: "CheckNumerics"
  input: "Identity_203"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_198/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_204"
  op: "Identity"
  input: "Read_198/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_198"
  op: "CheckNumerics"
  input: "Identity_204"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator/encoder/layer_11/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_199/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_205"
  op: "Identity"
  input: "Read_199/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_199"
  op: "CheckNumerics"
  input: "Identity_205"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 256
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator_predictions/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_200/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_206"
  op: "Identity"
  input: "Read_200/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_200"
  op: "CheckNumerics"
  input: "Identity_206"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator_predictions/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_201/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_207"
  op: "Identity"
  input: "Read_201/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_201"
  op: "CheckNumerics"
  input: "Identity_207"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator_predictions/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_202/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_208"
  op: "Identity"
  input: "Read_202/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_202"
  op: "CheckNumerics"
  input: "Identity_208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator_predictions/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_203/ReadVariableOp"
  op: "ReadVariableOp"
  input: "generator_predictions/output_bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_209"
  op: "Identity"
  input: "Read_203/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_203"
  op: "CheckNumerics"
  input: "Identity_209"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 50265
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for generator_predictions/output_bias:0 is NaN"
    }
  }
}
node {
  name: "Read_204/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_210"
  op: "Identity"
  input: "Read_204/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_204"
  op: "CheckNumerics"
  input: "Identity_210"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/embeddings_3/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_205/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_211"
  op: "Identity"
  input: "Read_205/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_205"
  op: "CheckNumerics"
  input: "Identity_211"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/embeddings_3/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_206/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_212"
  op: "Identity"
  input: "Read_206/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_206"
  op: "CheckNumerics"
  input: "Identity_212"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_207/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_213"
  op: "Identity"
  input: "Read_207/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_207"
  op: "CheckNumerics"
  input: "Identity_213"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_208/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_214"
  op: "Identity"
  input: "Read_208/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_208"
  op: "CheckNumerics"
  input: "Identity_214"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_209/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_215"
  op: "Identity"
  input: "Read_209/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_209"
  op: "CheckNumerics"
  input: "Identity_215"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_210/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_216"
  op: "Identity"
  input: "Read_210/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_210"
  op: "CheckNumerics"
  input: "Identity_216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_211/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_217"
  op: "Identity"
  input: "Read_211/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_211"
  op: "CheckNumerics"
  input: "Identity_217"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_212/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_218"
  op: "Identity"
  input: "Read_212/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_212"
  op: "CheckNumerics"
  input: "Identity_218"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_213/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_219"
  op: "Identity"
  input: "Read_213/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_213"
  op: "CheckNumerics"
  input: "Identity_219"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_214/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_220"
  op: "Identity"
  input: "Read_214/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_214"
  op: "CheckNumerics"
  input: "Identity_220"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_215/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_221"
  op: "Identity"
  input: "Read_215/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_215"
  op: "CheckNumerics"
  input: "Identity_221"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_216/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_222"
  op: "Identity"
  input: "Read_216/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_216"
  op: "CheckNumerics"
  input: "Identity_222"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_217/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_223"
  op: "Identity"
  input: "Read_217/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_217"
  op: "CheckNumerics"
  input: "Identity_223"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_218/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_224"
  op: "Identity"
  input: "Read_218/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_218"
  op: "CheckNumerics"
  input: "Identity_224"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_219/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_225"
  op: "Identity"
  input: "Read_219/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_219"
  op: "CheckNumerics"
  input: "Identity_225"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_220/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_226"
  op: "Identity"
  input: "Read_220/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_220"
  op: "CheckNumerics"
  input: "Identity_226"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_221/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_227"
  op: "Identity"
  input: "Read_221/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_221"
  op: "CheckNumerics"
  input: "Identity_227"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_0/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_222/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_228"
  op: "Identity"
  input: "Read_222/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_222"
  op: "CheckNumerics"
  input: "Identity_228"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_223/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_229"
  op: "Identity"
  input: "Read_223/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_223"
  op: "CheckNumerics"
  input: "Identity_229"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_224/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_230"
  op: "Identity"
  input: "Read_224/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_224"
  op: "CheckNumerics"
  input: "Identity_230"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_225/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_231"
  op: "Identity"
  input: "Read_225/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_225"
  op: "CheckNumerics"
  input: "Identity_231"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_226/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_232"
  op: "Identity"
  input: "Read_226/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_226"
  op: "CheckNumerics"
  input: "Identity_232"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_227/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_233"
  op: "Identity"
  input: "Read_227/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_227"
  op: "CheckNumerics"
  input: "Identity_233"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_228/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_234"
  op: "Identity"
  input: "Read_228/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_228"
  op: "CheckNumerics"
  input: "Identity_234"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_229/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_235"
  op: "Identity"
  input: "Read_229/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_229"
  op: "CheckNumerics"
  input: "Identity_235"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_230/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_236"
  op: "Identity"
  input: "Read_230/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_230"
  op: "CheckNumerics"
  input: "Identity_236"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_231/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_237"
  op: "Identity"
  input: "Read_231/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_231"
  op: "CheckNumerics"
  input: "Identity_237"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_232/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_238"
  op: "Identity"
  input: "Read_232/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_232"
  op: "CheckNumerics"
  input: "Identity_238"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_233/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_239"
  op: "Identity"
  input: "Read_233/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_233"
  op: "CheckNumerics"
  input: "Identity_239"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_234/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_240"
  op: "Identity"
  input: "Read_234/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_234"
  op: "CheckNumerics"
  input: "Identity_240"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_235/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_241"
  op: "Identity"
  input: "Read_235/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_235"
  op: "CheckNumerics"
  input: "Identity_241"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_236/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_242"
  op: "Identity"
  input: "Read_236/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_236"
  op: "CheckNumerics"
  input: "Identity_242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_237/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_243"
  op: "Identity"
  input: "Read_237/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_237"
  op: "CheckNumerics"
  input: "Identity_243"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_1/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_238/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_244"
  op: "Identity"
  input: "Read_238/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_238"
  op: "CheckNumerics"
  input: "Identity_244"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_239/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_245"
  op: "Identity"
  input: "Read_239/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_239"
  op: "CheckNumerics"
  input: "Identity_245"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_240/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_246"
  op: "Identity"
  input: "Read_240/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_240"
  op: "CheckNumerics"
  input: "Identity_246"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_241/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_247"
  op: "Identity"
  input: "Read_241/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_241"
  op: "CheckNumerics"
  input: "Identity_247"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_242/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_248"
  op: "Identity"
  input: "Read_242/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_242"
  op: "CheckNumerics"
  input: "Identity_248"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_243/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_249"
  op: "Identity"
  input: "Read_243/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_243"
  op: "CheckNumerics"
  input: "Identity_249"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_244/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_250"
  op: "Identity"
  input: "Read_244/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_244"
  op: "CheckNumerics"
  input: "Identity_250"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_245/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_251"
  op: "Identity"
  input: "Read_245/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_245"
  op: "CheckNumerics"
  input: "Identity_251"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_246/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_252"
  op: "Identity"
  input: "Read_246/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_246"
  op: "CheckNumerics"
  input: "Identity_252"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_247/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_253"
  op: "Identity"
  input: "Read_247/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_247"
  op: "CheckNumerics"
  input: "Identity_253"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_248/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_254"
  op: "Identity"
  input: "Read_248/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_248"
  op: "CheckNumerics"
  input: "Identity_254"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_249/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_255"
  op: "Identity"
  input: "Read_249/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_249"
  op: "CheckNumerics"
  input: "Identity_255"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_250/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_256"
  op: "Identity"
  input: "Read_250/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_250"
  op: "CheckNumerics"
  input: "Identity_256"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_251/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_257"
  op: "Identity"
  input: "Read_251/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_251"
  op: "CheckNumerics"
  input: "Identity_257"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_252/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_258"
  op: "Identity"
  input: "Read_252/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_252"
  op: "CheckNumerics"
  input: "Identity_258"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_253/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_259"
  op: "Identity"
  input: "Read_253/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_253"
  op: "CheckNumerics"
  input: "Identity_259"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_2/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_254/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_260"
  op: "Identity"
  input: "Read_254/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_254"
  op: "CheckNumerics"
  input: "Identity_260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_255/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_261"
  op: "Identity"
  input: "Read_255/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_255"
  op: "CheckNumerics"
  input: "Identity_261"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_256/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_262"
  op: "Identity"
  input: "Read_256/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_256"
  op: "CheckNumerics"
  input: "Identity_262"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_257/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_263"
  op: "Identity"
  input: "Read_257/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_257"
  op: "CheckNumerics"
  input: "Identity_263"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_258/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_264"
  op: "Identity"
  input: "Read_258/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_258"
  op: "CheckNumerics"
  input: "Identity_264"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_259/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_265"
  op: "Identity"
  input: "Read_259/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_259"
  op: "CheckNumerics"
  input: "Identity_265"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_260/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_266"
  op: "Identity"
  input: "Read_260/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_260"
  op: "CheckNumerics"
  input: "Identity_266"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_261/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_267"
  op: "Identity"
  input: "Read_261/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_261"
  op: "CheckNumerics"
  input: "Identity_267"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_262/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_268"
  op: "Identity"
  input: "Read_262/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_262"
  op: "CheckNumerics"
  input: "Identity_268"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_263/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_269"
  op: "Identity"
  input: "Read_263/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_263"
  op: "CheckNumerics"
  input: "Identity_269"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_264/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_270"
  op: "Identity"
  input: "Read_264/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_264"
  op: "CheckNumerics"
  input: "Identity_270"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_265/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_271"
  op: "Identity"
  input: "Read_265/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_265"
  op: "CheckNumerics"
  input: "Identity_271"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_266/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_272"
  op: "Identity"
  input: "Read_266/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_266"
  op: "CheckNumerics"
  input: "Identity_272"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_267/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_273"
  op: "Identity"
  input: "Read_267/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_267"
  op: "CheckNumerics"
  input: "Identity_273"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_268/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_274"
  op: "Identity"
  input: "Read_268/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_268"
  op: "CheckNumerics"
  input: "Identity_274"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_269/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_275"
  op: "Identity"
  input: "Read_269/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_269"
  op: "CheckNumerics"
  input: "Identity_275"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_3/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_270/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_276"
  op: "Identity"
  input: "Read_270/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_270"
  op: "CheckNumerics"
  input: "Identity_276"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_271/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_277"
  op: "Identity"
  input: "Read_271/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_271"
  op: "CheckNumerics"
  input: "Identity_277"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_272/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_278"
  op: "Identity"
  input: "Read_272/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_272"
  op: "CheckNumerics"
  input: "Identity_278"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_273/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_279"
  op: "Identity"
  input: "Read_273/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_273"
  op: "CheckNumerics"
  input: "Identity_279"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_274/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_280"
  op: "Identity"
  input: "Read_274/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_274"
  op: "CheckNumerics"
  input: "Identity_280"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_275/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_281"
  op: "Identity"
  input: "Read_275/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_275"
  op: "CheckNumerics"
  input: "Identity_281"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_276/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_282"
  op: "Identity"
  input: "Read_276/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_276"
  op: "CheckNumerics"
  input: "Identity_282"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_277/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_283"
  op: "Identity"
  input: "Read_277/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_277"
  op: "CheckNumerics"
  input: "Identity_283"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_278/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_284"
  op: "Identity"
  input: "Read_278/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_278"
  op: "CheckNumerics"
  input: "Identity_284"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_279/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_285"
  op: "Identity"
  input: "Read_279/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_279"
  op: "CheckNumerics"
  input: "Identity_285"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_280/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_286"
  op: "Identity"
  input: "Read_280/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_280"
  op: "CheckNumerics"
  input: "Identity_286"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_281/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_287"
  op: "Identity"
  input: "Read_281/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_281"
  op: "CheckNumerics"
  input: "Identity_287"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_282/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_288"
  op: "Identity"
  input: "Read_282/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_282"
  op: "CheckNumerics"
  input: "Identity_288"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_283/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_289"
  op: "Identity"
  input: "Read_283/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_283"
  op: "CheckNumerics"
  input: "Identity_289"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_284/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_290"
  op: "Identity"
  input: "Read_284/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_284"
  op: "CheckNumerics"
  input: "Identity_290"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_285/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_291"
  op: "Identity"
  input: "Read_285/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_285"
  op: "CheckNumerics"
  input: "Identity_291"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_4/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_286/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_292"
  op: "Identity"
  input: "Read_286/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_286"
  op: "CheckNumerics"
  input: "Identity_292"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_287/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_293"
  op: "Identity"
  input: "Read_287/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_287"
  op: "CheckNumerics"
  input: "Identity_293"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_288/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_294"
  op: "Identity"
  input: "Read_288/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_288"
  op: "CheckNumerics"
  input: "Identity_294"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_289/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_295"
  op: "Identity"
  input: "Read_289/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_289"
  op: "CheckNumerics"
  input: "Identity_295"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_290/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_296"
  op: "Identity"
  input: "Read_290/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_290"
  op: "CheckNumerics"
  input: "Identity_296"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_291/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_297"
  op: "Identity"
  input: "Read_291/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_291"
  op: "CheckNumerics"
  input: "Identity_297"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_292/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_298"
  op: "Identity"
  input: "Read_292/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_292"
  op: "CheckNumerics"
  input: "Identity_298"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_293/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_299"
  op: "Identity"
  input: "Read_293/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_293"
  op: "CheckNumerics"
  input: "Identity_299"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_294/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_300"
  op: "Identity"
  input: "Read_294/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_294"
  op: "CheckNumerics"
  input: "Identity_300"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_295/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_301"
  op: "Identity"
  input: "Read_295/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_295"
  op: "CheckNumerics"
  input: "Identity_301"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_296/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_302"
  op: "Identity"
  input: "Read_296/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_296"
  op: "CheckNumerics"
  input: "Identity_302"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_297/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_303"
  op: "Identity"
  input: "Read_297/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_297"
  op: "CheckNumerics"
  input: "Identity_303"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_298/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_304"
  op: "Identity"
  input: "Read_298/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_298"
  op: "CheckNumerics"
  input: "Identity_304"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_299/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_305"
  op: "Identity"
  input: "Read_299/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_299"
  op: "CheckNumerics"
  input: "Identity_305"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_300/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_306"
  op: "Identity"
  input: "Read_300/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_300"
  op: "CheckNumerics"
  input: "Identity_306"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_301/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_307"
  op: "Identity"
  input: "Read_301/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_301"
  op: "CheckNumerics"
  input: "Identity_307"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_5/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_302/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_308"
  op: "Identity"
  input: "Read_302/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_302"
  op: "CheckNumerics"
  input: "Identity_308"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_303/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_309"
  op: "Identity"
  input: "Read_303/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_303"
  op: "CheckNumerics"
  input: "Identity_309"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_304/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_310"
  op: "Identity"
  input: "Read_304/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_304"
  op: "CheckNumerics"
  input: "Identity_310"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_305/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_311"
  op: "Identity"
  input: "Read_305/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_305"
  op: "CheckNumerics"
  input: "Identity_311"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_306/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_312"
  op: "Identity"
  input: "Read_306/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_306"
  op: "CheckNumerics"
  input: "Identity_312"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_307/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_313"
  op: "Identity"
  input: "Read_307/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_307"
  op: "CheckNumerics"
  input: "Identity_313"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_308/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_314"
  op: "Identity"
  input: "Read_308/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_308"
  op: "CheckNumerics"
  input: "Identity_314"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_309/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_315"
  op: "Identity"
  input: "Read_309/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_309"
  op: "CheckNumerics"
  input: "Identity_315"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_310/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_316"
  op: "Identity"
  input: "Read_310/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_310"
  op: "CheckNumerics"
  input: "Identity_316"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_311/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_317"
  op: "Identity"
  input: "Read_311/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_311"
  op: "CheckNumerics"
  input: "Identity_317"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_312/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_318"
  op: "Identity"
  input: "Read_312/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_312"
  op: "CheckNumerics"
  input: "Identity_318"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_313/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_319"
  op: "Identity"
  input: "Read_313/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_313"
  op: "CheckNumerics"
  input: "Identity_319"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_314/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_320"
  op: "Identity"
  input: "Read_314/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_314"
  op: "CheckNumerics"
  input: "Identity_320"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_315/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_321"
  op: "Identity"
  input: "Read_315/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_315"
  op: "CheckNumerics"
  input: "Identity_321"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_316/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_322"
  op: "Identity"
  input: "Read_316/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_316"
  op: "CheckNumerics"
  input: "Identity_322"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_317/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_323"
  op: "Identity"
  input: "Read_317/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_317"
  op: "CheckNumerics"
  input: "Identity_323"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_6/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_318/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_324"
  op: "Identity"
  input: "Read_318/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_318"
  op: "CheckNumerics"
  input: "Identity_324"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_319/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_325"
  op: "Identity"
  input: "Read_319/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_319"
  op: "CheckNumerics"
  input: "Identity_325"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_320/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_326"
  op: "Identity"
  input: "Read_320/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_320"
  op: "CheckNumerics"
  input: "Identity_326"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_321/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_327"
  op: "Identity"
  input: "Read_321/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_321"
  op: "CheckNumerics"
  input: "Identity_327"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_322/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_328"
  op: "Identity"
  input: "Read_322/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_322"
  op: "CheckNumerics"
  input: "Identity_328"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_323/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_329"
  op: "Identity"
  input: "Read_323/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_323"
  op: "CheckNumerics"
  input: "Identity_329"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_324/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_330"
  op: "Identity"
  input: "Read_324/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_324"
  op: "CheckNumerics"
  input: "Identity_330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_325/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_331"
  op: "Identity"
  input: "Read_325/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_325"
  op: "CheckNumerics"
  input: "Identity_331"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_326/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_332"
  op: "Identity"
  input: "Read_326/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_326"
  op: "CheckNumerics"
  input: "Identity_332"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_327/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_333"
  op: "Identity"
  input: "Read_327/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_327"
  op: "CheckNumerics"
  input: "Identity_333"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_328/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_334"
  op: "Identity"
  input: "Read_328/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_328"
  op: "CheckNumerics"
  input: "Identity_334"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_329/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_335"
  op: "Identity"
  input: "Read_329/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_329"
  op: "CheckNumerics"
  input: "Identity_335"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_330/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_336"
  op: "Identity"
  input: "Read_330/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_330"
  op: "CheckNumerics"
  input: "Identity_336"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_331/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_337"
  op: "Identity"
  input: "Read_331/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_331"
  op: "CheckNumerics"
  input: "Identity_337"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_332/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_338"
  op: "Identity"
  input: "Read_332/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_332"
  op: "CheckNumerics"
  input: "Identity_338"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_333/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_339"
  op: "Identity"
  input: "Read_333/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_333"
  op: "CheckNumerics"
  input: "Identity_339"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_7/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_334/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_340"
  op: "Identity"
  input: "Read_334/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_334"
  op: "CheckNumerics"
  input: "Identity_340"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_335/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_341"
  op: "Identity"
  input: "Read_335/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_335"
  op: "CheckNumerics"
  input: "Identity_341"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_336/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_342"
  op: "Identity"
  input: "Read_336/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_336"
  op: "CheckNumerics"
  input: "Identity_342"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_337/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_343"
  op: "Identity"
  input: "Read_337/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_337"
  op: "CheckNumerics"
  input: "Identity_343"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_338/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_344"
  op: "Identity"
  input: "Read_338/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_338"
  op: "CheckNumerics"
  input: "Identity_344"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_339/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_345"
  op: "Identity"
  input: "Read_339/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_339"
  op: "CheckNumerics"
  input: "Identity_345"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_340/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_346"
  op: "Identity"
  input: "Read_340/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_340"
  op: "CheckNumerics"
  input: "Identity_346"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_341/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_347"
  op: "Identity"
  input: "Read_341/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_341"
  op: "CheckNumerics"
  input: "Identity_347"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_342/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_348"
  op: "Identity"
  input: "Read_342/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_342"
  op: "CheckNumerics"
  input: "Identity_348"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_343/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_349"
  op: "Identity"
  input: "Read_343/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_343"
  op: "CheckNumerics"
  input: "Identity_349"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_344/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_350"
  op: "Identity"
  input: "Read_344/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_344"
  op: "CheckNumerics"
  input: "Identity_350"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_345/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_351"
  op: "Identity"
  input: "Read_345/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_345"
  op: "CheckNumerics"
  input: "Identity_351"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_346/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_352"
  op: "Identity"
  input: "Read_346/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_346"
  op: "CheckNumerics"
  input: "Identity_352"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_347/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_353"
  op: "Identity"
  input: "Read_347/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_347"
  op: "CheckNumerics"
  input: "Identity_353"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_348/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_354"
  op: "Identity"
  input: "Read_348/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_348"
  op: "CheckNumerics"
  input: "Identity_354"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_349/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_355"
  op: "Identity"
  input: "Read_349/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_349"
  op: "CheckNumerics"
  input: "Identity_355"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_8/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_350/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_356"
  op: "Identity"
  input: "Read_350/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_350"
  op: "CheckNumerics"
  input: "Identity_356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_351/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_357"
  op: "Identity"
  input: "Read_351/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_351"
  op: "CheckNumerics"
  input: "Identity_357"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_352/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_358"
  op: "Identity"
  input: "Read_352/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_352"
  op: "CheckNumerics"
  input: "Identity_358"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_353/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_359"
  op: "Identity"
  input: "Read_353/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_353"
  op: "CheckNumerics"
  input: "Identity_359"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_354/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_360"
  op: "Identity"
  input: "Read_354/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_354"
  op: "CheckNumerics"
  input: "Identity_360"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_355/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_361"
  op: "Identity"
  input: "Read_355/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_355"
  op: "CheckNumerics"
  input: "Identity_361"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_356/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_362"
  op: "Identity"
  input: "Read_356/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_356"
  op: "CheckNumerics"
  input: "Identity_362"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_357/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_363"
  op: "Identity"
  input: "Read_357/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_357"
  op: "CheckNumerics"
  input: "Identity_363"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_358/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_364"
  op: "Identity"
  input: "Read_358/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_358"
  op: "CheckNumerics"
  input: "Identity_364"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_359/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_365"
  op: "Identity"
  input: "Read_359/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_359"
  op: "CheckNumerics"
  input: "Identity_365"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_360/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_366"
  op: "Identity"
  input: "Read_360/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_360"
  op: "CheckNumerics"
  input: "Identity_366"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_361/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_367"
  op: "Identity"
  input: "Read_361/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_361"
  op: "CheckNumerics"
  input: "Identity_367"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_362/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_368"
  op: "Identity"
  input: "Read_362/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_362"
  op: "CheckNumerics"
  input: "Identity_368"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_363/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_369"
  op: "Identity"
  input: "Read_363/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_363"
  op: "CheckNumerics"
  input: "Identity_369"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_364/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_370"
  op: "Identity"
  input: "Read_364/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_364"
  op: "CheckNumerics"
  input: "Identity_370"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_365/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_371"
  op: "Identity"
  input: "Read_365/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_365"
  op: "CheckNumerics"
  input: "Identity_371"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_9/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_366/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_372"
  op: "Identity"
  input: "Read_366/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_366"
  op: "CheckNumerics"
  input: "Identity_372"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_367/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_373"
  op: "Identity"
  input: "Read_367/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_367"
  op: "CheckNumerics"
  input: "Identity_373"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_368/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_374"
  op: "Identity"
  input: "Read_368/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_368"
  op: "CheckNumerics"
  input: "Identity_374"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_369/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_375"
  op: "Identity"
  input: "Read_369/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_369"
  op: "CheckNumerics"
  input: "Identity_375"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_370/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_376"
  op: "Identity"
  input: "Read_370/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_370"
  op: "CheckNumerics"
  input: "Identity_376"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_371/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_377"
  op: "Identity"
  input: "Read_371/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_371"
  op: "CheckNumerics"
  input: "Identity_377"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_372/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_378"
  op: "Identity"
  input: "Read_372/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_372"
  op: "CheckNumerics"
  input: "Identity_378"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_373/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_379"
  op: "Identity"
  input: "Read_373/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_373"
  op: "CheckNumerics"
  input: "Identity_379"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_374/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_380"
  op: "Identity"
  input: "Read_374/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_374"
  op: "CheckNumerics"
  input: "Identity_380"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_375/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_381"
  op: "Identity"
  input: "Read_375/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_375"
  op: "CheckNumerics"
  input: "Identity_381"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_376/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_382"
  op: "Identity"
  input: "Read_376/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_376"
  op: "CheckNumerics"
  input: "Identity_382"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_377/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_383"
  op: "Identity"
  input: "Read_377/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_377"
  op: "CheckNumerics"
  input: "Identity_383"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_378/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_384"
  op: "Identity"
  input: "Read_378/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_378"
  op: "CheckNumerics"
  input: "Identity_384"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_379/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_385"
  op: "Identity"
  input: "Read_379/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_379"
  op: "CheckNumerics"
  input: "Identity_385"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_380/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_386"
  op: "Identity"
  input: "Read_380/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_380"
  op: "CheckNumerics"
  input: "Identity_386"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_381/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_387"
  op: "Identity"
  input: "Read_381/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_381"
  op: "CheckNumerics"
  input: "Identity_387"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_10/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_382/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_388"
  op: "Identity"
  input: "Read_382/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_382"
  op: "CheckNumerics"
  input: "Identity_388"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/attention/self/query/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_383/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_389"
  op: "Identity"
  input: "Read_383/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_383"
  op: "CheckNumerics"
  input: "Identity_389"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/attention/self/query/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_384/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_390"
  op: "Identity"
  input: "Read_384/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_384"
  op: "CheckNumerics"
  input: "Identity_390"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/attention/self/key/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_385/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_391"
  op: "Identity"
  input: "Read_385/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_385"
  op: "CheckNumerics"
  input: "Identity_391"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/attention/self/key/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_386/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_392"
  op: "Identity"
  input: "Read_386/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_386"
  op: "CheckNumerics"
  input: "Identity_392"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/attention/self/value/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_387/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_393"
  op: "Identity"
  input: "Read_387/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_387"
  op: "CheckNumerics"
  input: "Identity_393"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/attention/self/value/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_388/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_394"
  op: "Identity"
  input: "Read_388/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_388"
  op: "CheckNumerics"
  input: "Identity_394"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/attention/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_389/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_395"
  op: "Identity"
  input: "Read_389/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_389"
  op: "CheckNumerics"
  input: "Identity_395"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/attention/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_390/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_396"
  op: "Identity"
  input: "Read_390/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_390"
  op: "CheckNumerics"
  input: "Identity_396"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/attention/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_391/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_397"
  op: "Identity"
  input: "Read_391/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_391"
  op: "CheckNumerics"
  input: "Identity_397"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/attention/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_392/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_398"
  op: "Identity"
  input: "Read_392/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_392"
  op: "CheckNumerics"
  input: "Identity_398"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/intermediate/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_393/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_399"
  op: "Identity"
  input: "Read_393/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_393"
  op: "CheckNumerics"
  input: "Identity_399"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/intermediate/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_394/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_400"
  op: "Identity"
  input: "Read_394/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_394"
  op: "CheckNumerics"
  input: "Identity_400"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/output/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_395/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_401"
  op: "Identity"
  input: "Read_395/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_395"
  op: "CheckNumerics"
  input: "Identity_401"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/output/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_396/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_402"
  op: "Identity"
  input: "Read_396/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_396"
  op: "CheckNumerics"
  input: "Identity_402"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/output/LayerNorm/gamma:0 is NaN"
    }
  }
}
node {
  name: "Read_397/ReadVariableOp"
  op: "ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_403"
  op: "Identity"
  input: "Read_397/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_397"
  op: "CheckNumerics"
  input: "Identity_403"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for electra/encoder/layer_11/output/LayerNorm/beta:0 is NaN"
    }
  }
}
node {
  name: "Read_398/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_404"
  op: "Identity"
  input: "Read_398/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_398"
  op: "CheckNumerics"
  input: "Identity_404"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for discriminator_predictions/dense/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_399/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_405"
  op: "Identity"
  input: "Read_399/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_399"
  op: "CheckNumerics"
  input: "Identity_405"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for discriminator_predictions/dense/bias:0 is NaN"
    }
  }
}
node {
  name: "Read_400/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/kernel"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_406"
  op: "Identity"
  input: "Read_400/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_400"
  op: "CheckNumerics"
  input: "Identity_406"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for discriminator_predictions/dense_1/kernel:0 is NaN"
    }
  }
}
node {
  name: "Read_401/ReadVariableOp"
  op: "ReadVariableOp"
  input: "discriminator_predictions/dense_1/bias"
  input: "^output_0_shard_0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "Identity_407"
  op: "Identity"
  input: "Read_401/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "CheckNumerics_401"
  op: "CheckNumerics"
  input: "Identity_407"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "message"
    value {
      s: "Gradient for discriminator_predictions/dense_1/bias:0 is NaN"
    }
  }
}
node {
  name: "group_deps_3"
  op: "NoOp"
  input: "^CheckNumerics"
  input: "^CheckNumerics_1"
  input: "^CheckNumerics_10"
  input: "^CheckNumerics_100"
  input: "^CheckNumerics_101"
  input: "^CheckNumerics_102"
  input: "^CheckNumerics_103"
  input: "^CheckNumerics_104"
  input: "^CheckNumerics_105"
  input: "^CheckNumerics_106"
  input: "^CheckNumerics_107"
  input: "^CheckNumerics_108"
  input: "^CheckNumerics_109"
  input: "^CheckNumerics_11"
  input: "^CheckNumerics_110"
  input: "^CheckNumerics_111"
  input: "^CheckNumerics_112"
  input: "^CheckNumerics_113"
  input: "^CheckNumerics_114"
  input: "^CheckNumerics_115"
  input: "^CheckNumerics_116"
  input: "^CheckNumerics_117"
  input: "^CheckNumerics_118"
  input: "^CheckNumerics_119"
  input: "^CheckNumerics_12"
  input: "^CheckNumerics_120"
  input: "^CheckNumerics_121"
  input: "^CheckNumerics_122"
  input: "^CheckNumerics_123"
  input: "^CheckNumerics_124"
  input: "^CheckNumerics_125"
  input: "^CheckNumerics_126"
  input: "^CheckNumerics_127"
  input: "^CheckNumerics_128"
  input: "^CheckNumerics_129"
  input: "^CheckNumerics_13"
  input: "^CheckNumerics_130"
  input: "^CheckNumerics_131"
  input: "^CheckNumerics_132"
  input: "^CheckNumerics_133"
  input: "^CheckNumerics_134"
  input: "^CheckNumerics_135"
  input: "^CheckNumerics_136"
  input: "^CheckNumerics_137"
  input: "^CheckNumerics_138"
  input: "^CheckNumerics_139"
  input: "^CheckNumerics_14"
  input: "^CheckNumerics_140"
  input: "^CheckNumerics_141"
  input: "^CheckNumerics_142"
  input: "^CheckNumerics_143"
  input: "^CheckNumerics_144"
  input: "^CheckNumerics_145"
  input: "^CheckNumerics_146"
  input: "^CheckNumerics_147"
  input: "^CheckNumerics_148"
  input: "^CheckNumerics_149"
  input: "^CheckNumerics_15"
  input: "^CheckNumerics_150"
  input: "^CheckNumerics_151"
  input: "^CheckNumerics_152"
  input: "^CheckNumerics_153"
  input: "^CheckNumerics_154"
  input: "^CheckNumerics_155"
  input: "^CheckNumerics_156"
  input: "^CheckNumerics_157"
  input: "^CheckNumerics_158"
  input: "^CheckNumerics_159"
  input: "^CheckNumerics_16"
  input: "^CheckNumerics_160"
  input: "^CheckNumerics_161"
  input: "^CheckNumerics_162"
  input: "^CheckNumerics_163"
  input: "^CheckNumerics_164"
  input: "^CheckNumerics_165"
  input: "^CheckNumerics_166"
  input: "^CheckNumerics_167"
  input: "^CheckNumerics_168"
  input: "^CheckNumerics_169"
  input: "^CheckNumerics_17"
  input: "^CheckNumerics_170"
  input: "^CheckNumerics_171"
  input: "^CheckNumerics_172"
  input: "^CheckNumerics_173"
  input: "^CheckNumerics_174"
  input: "^CheckNumerics_175"
  input: "^CheckNumerics_176"
  input: "^CheckNumerics_177"
  input: "^CheckNumerics_178"
  input: "^CheckNumerics_179"
  input: "^CheckNumerics_18"
  input: "^CheckNumerics_180"
  input: "^CheckNumerics_181"
  input: "^CheckNumerics_182"
  input: "^CheckNumerics_183"
  input: "^CheckNumerics_184"
  input: "^CheckNumerics_185"
  input: "^CheckNumerics_186"
  input: "^CheckNumerics_187"
  input: "^CheckNumerics_188"
  input: "^CheckNumerics_189"
  input: "^CheckNumerics_19"
  input: "^CheckNumerics_190"
  input: "^CheckNumerics_191"
  input: "^CheckNumerics_192"
  input: "^CheckNumerics_193"
  input: "^CheckNumerics_194"
  input: "^CheckNumerics_195"
  input: "^CheckNumerics_196"
  input: "^CheckNumerics_197"
  input: "^CheckNumerics_198"
  input: "^CheckNumerics_199"
  input: "^CheckNumerics_2"
  input: "^CheckNumerics_20"
  input: "^CheckNumerics_200"
  input: "^CheckNumerics_201"
  input: "^CheckNumerics_202"
  input: "^CheckNumerics_203"
  input: "^CheckNumerics_204"
  input: "^CheckNumerics_205"
  input: "^CheckNumerics_206"
  input: "^CheckNumerics_207"
  input: "^CheckNumerics_208"
  input: "^CheckNumerics_209"
  input: "^CheckNumerics_21"
  input: "^CheckNumerics_210"
  input: "^CheckNumerics_211"
  input: "^CheckNumerics_212"
  input: "^CheckNumerics_213"
  input: "^CheckNumerics_214"
  input: "^CheckNumerics_215"
  input: "^CheckNumerics_216"
  input: "^CheckNumerics_217"
  input: "^CheckNumerics_218"
  input: "^CheckNumerics_219"
  input: "^CheckNumerics_22"
  input: "^CheckNumerics_220"
  input: "^CheckNumerics_221"
  input: "^CheckNumerics_222"
  input: "^CheckNumerics_223"
  input: "^CheckNumerics_224"
  input: "^CheckNumerics_225"
  input: "^CheckNumerics_226"
  input: "^CheckNumerics_227"
  input: "^CheckNumerics_228"
  input: "^CheckNumerics_229"
  input: "^CheckNumerics_23"
  input: "^CheckNumerics_230"
  input: "^CheckNumerics_231"
  input: "^CheckNumerics_232"
  input: "^CheckNumerics_233"
  input: "^CheckNumerics_234"
  input: "^CheckNumerics_235"
  input: "^CheckNumerics_236"
  input: "^CheckNumerics_237"
  input: "^CheckNumerics_238"
  input: "^CheckNumerics_239"
  input: "^CheckNumerics_24"
  input: "^CheckNumerics_240"
  input: "^CheckNumerics_241"
  input: "^CheckNumerics_242"
  input: "^CheckNumerics_243"
  input: "^CheckNumerics_244"
  input: "^CheckNumerics_245"
  input: "^CheckNumerics_246"
  input: "^CheckNumerics_247"
  input: "^CheckNumerics_248"
  input: "^CheckNumerics_249"
  input: "^CheckNumerics_25"
  input: "^CheckNumerics_250"
  input: "^CheckNumerics_251"
  input: "^CheckNumerics_252"
  input: "^CheckNumerics_253"
  input: "^CheckNumerics_254"
  input: "^CheckNumerics_255"
  input: "^CheckNumerics_256"
  input: "^CheckNumerics_257"
  input: "^CheckNumerics_258"
  input: "^CheckNumerics_259"
  input: "^CheckNumerics_26"
  input: "^CheckNumerics_260"
  input: "^CheckNumerics_261"
  input: "^CheckNumerics_262"
  input: "^CheckNumerics_263"
  input: "^CheckNumerics_264"
  input: "^CheckNumerics_265"
  input: "^CheckNumerics_266"
  input: "^CheckNumerics_267"
  input: "^CheckNumerics_268"
  input: "^CheckNumerics_269"
  input: "^CheckNumerics_27"
  input: "^CheckNumerics_270"
  input: "^CheckNumerics_271"
  input: "^CheckNumerics_272"
  input: "^CheckNumerics_273"
  input: "^CheckNumerics_274"
  input: "^CheckNumerics_275"
  input: "^CheckNumerics_276"
  input: "^CheckNumerics_277"
  input: "^CheckNumerics_278"
  input: "^CheckNumerics_279"
  input: "^CheckNumerics_28"
  input: "^CheckNumerics_280"
  input: "^CheckNumerics_281"
  input: "^CheckNumerics_282"
  input: "^CheckNumerics_283"
  input: "^CheckNumerics_284"
  input: "^CheckNumerics_285"
  input: "^CheckNumerics_286"
  input: "^CheckNumerics_287"
  input: "^CheckNumerics_288"
  input: "^CheckNumerics_289"
  input: "^CheckNumerics_29"
  input: "^CheckNumerics_290"
  input: "^CheckNumerics_291"
  input: "^CheckNumerics_292"
  input: "^CheckNumerics_293"
  input: "^CheckNumerics_294"
  input: "^CheckNumerics_295"
  input: "^CheckNumerics_296"
  input: "^CheckNumerics_297"
  input: "^CheckNumerics_298"
  input: "^CheckNumerics_299"
  input: "^CheckNumerics_3"
  input: "^CheckNumerics_30"
  input: "^CheckNumerics_300"
  input: "^CheckNumerics_301"
  input: "^CheckNumerics_302"
  input: "^CheckNumerics_303"
  input: "^CheckNumerics_304"
  input: "^CheckNumerics_305"
  input: "^CheckNumerics_306"
  input: "^CheckNumerics_307"
  input: "^CheckNumerics_308"
  input: "^CheckNumerics_309"
  input: "^CheckNumerics_31"
  input: "^CheckNumerics_310"
  input: "^CheckNumerics_311"
  input: "^CheckNumerics_312"
  input: "^CheckNumerics_313"
  input: "^CheckNumerics_314"
  input: "^CheckNumerics_315"
  input: "^CheckNumerics_316"
  input: "^CheckNumerics_317"
  input: "^CheckNumerics_318"
  input: "^CheckNumerics_319"
  input: "^CheckNumerics_32"
  input: "^CheckNumerics_320"
  input: "^CheckNumerics_321"
  input: "^CheckNumerics_322"
  input: "^CheckNumerics_323"
  input: "^CheckNumerics_324"
  input: "^CheckNumerics_325"
  input: "^CheckNumerics_326"
  input: "^CheckNumerics_327"
  input: "^CheckNumerics_328"
  input: "^CheckNumerics_329"
  input: "^CheckNumerics_33"
  input: "^CheckNumerics_330"
  input: "^CheckNumerics_331"
  input: "^CheckNumerics_332"
  input: "^CheckNumerics_333"
  input: "^CheckNumerics_334"
  input: "^CheckNumerics_335"
  input: "^CheckNumerics_336"
  input: "^CheckNumerics_337"
  input: "^CheckNumerics_338"
  input: "^CheckNumerics_339"
  input: "^CheckNumerics_34"
  input: "^CheckNumerics_340"
  input: "^CheckNumerics_341"
  input: "^CheckNumerics_342"
  input: "^CheckNumerics_343"
  input: "^CheckNumerics_344"
  input: "^CheckNumerics_345"
  input: "^CheckNumerics_346"
  input: "^CheckNumerics_347"
  input: "^CheckNumerics_348"
  input: "^CheckNumerics_349"
  input: "^CheckNumerics_35"
  input: "^CheckNumerics_350"
  input: "^CheckNumerics_351"
  input: "^CheckNumerics_352"
  input: "^CheckNumerics_353"
  input: "^CheckNumerics_354"
  input: "^CheckNumerics_355"
  input: "^CheckNumerics_356"
  input: "^CheckNumerics_357"
  input: "^CheckNumerics_358"
  input: "^CheckNumerics_359"
  input: "^CheckNumerics_36"
  input: "^CheckNumerics_360"
  input: "^CheckNumerics_361"
  input: "^CheckNumerics_362"
  input: "^CheckNumerics_363"
  input: "^CheckNumerics_364"
  input: "^CheckNumerics_365"
  input: "^CheckNumerics_366"
  input: "^CheckNumerics_367"
  input: "^CheckNumerics_368"
  input: "^CheckNumerics_369"
  input: "^CheckNumerics_37"
  input: "^CheckNumerics_370"
  input: "^CheckNumerics_371"
  input: "^CheckNumerics_372"
  input: "^CheckNumerics_373"
  input: "^CheckNumerics_374"
  input: "^CheckNumerics_375"
  input: "^CheckNumerics_376"
  input: "^CheckNumerics_377"
  input: "^CheckNumerics_378"
  input: "^CheckNumerics_379"
  input: "^CheckNumerics_38"
  input: "^CheckNumerics_380"
  input: "^CheckNumerics_381"
  input: "^CheckNumerics_382"
  input: "^CheckNumerics_383"
  input: "^CheckNumerics_384"
  input: "^CheckNumerics_385"
  input: "^CheckNumerics_386"
  input: "^CheckNumerics_387"
  input: "^CheckNumerics_388"
  input: "^CheckNumerics_389"
  input: "^CheckNumerics_39"
  input: "^CheckNumerics_390"
  input: "^CheckNumerics_391"
  input: "^CheckNumerics_392"
  input: "^CheckNumerics_393"
  input: "^CheckNumerics_394"
  input: "^CheckNumerics_395"
  input: "^CheckNumerics_396"
  input: "^CheckNumerics_397"
  input: "^CheckNumerics_398"
  input: "^CheckNumerics_399"
  input: "^CheckNumerics_4"
  input: "^CheckNumerics_40"
  input: "^CheckNumerics_400"
  input: "^CheckNumerics_401"
  input: "^CheckNumerics_41"
  input: "^CheckNumerics_42"
  input: "^CheckNumerics_43"
  input: "^CheckNumerics_44"
  input: "^CheckNumerics_45"
  input: "^CheckNumerics_46"
  input: "^CheckNumerics_47"
  input: "^CheckNumerics_48"
  input: "^CheckNumerics_49"
  input: "^CheckNumerics_5"
  input: "^CheckNumerics_50"
  input: "^CheckNumerics_51"
  input: "^CheckNumerics_52"
  input: "^CheckNumerics_53"
  input: "^CheckNumerics_54"
  input: "^CheckNumerics_55"
  input: "^CheckNumerics_56"
  input: "^CheckNumerics_57"
  input: "^CheckNumerics_58"
  input: "^CheckNumerics_59"
  input: "^CheckNumerics_6"
  input: "^CheckNumerics_60"
  input: "^CheckNumerics_61"
  input: "^CheckNumerics_62"
  input: "^CheckNumerics_63"
  input: "^CheckNumerics_64"
  input: "^CheckNumerics_65"
  input: "^CheckNumerics_66"
  input: "^CheckNumerics_67"
  input: "^CheckNumerics_68"
  input: "^CheckNumerics_69"
  input: "^CheckNumerics_7"
  input: "^CheckNumerics_70"
  input: "^CheckNumerics_71"
  input: "^CheckNumerics_72"
  input: "^CheckNumerics_73"
  input: "^CheckNumerics_74"
  input: "^CheckNumerics_75"
  input: "^CheckNumerics_76"
  input: "^CheckNumerics_77"
  input: "^CheckNumerics_78"
  input: "^CheckNumerics_79"
  input: "^CheckNumerics_8"
  input: "^CheckNumerics_80"
  input: "^CheckNumerics_81"
  input: "^CheckNumerics_82"
  input: "^CheckNumerics_83"
  input: "^CheckNumerics_84"
  input: "^CheckNumerics_85"
  input: "^CheckNumerics_86"
  input: "^CheckNumerics_87"
  input: "^CheckNumerics_88"
  input: "^CheckNumerics_89"
  input: "^CheckNumerics_9"
  input: "^CheckNumerics_90"
  input: "^CheckNumerics_91"
  input: "^CheckNumerics_92"
  input: "^CheckNumerics_93"
  input: "^CheckNumerics_94"
  input: "^CheckNumerics_95"
  input: "^CheckNumerics_96"
  input: "^CheckNumerics_97"
  input: "^CheckNumerics_98"
  input: "^CheckNumerics_99"
}
node {
  name: "ShutdownDistributedTPU"
  op: "ShutdownDistributedTPU"
  device: "/device:TPU_SYSTEM:0"
}
node {
  name: "init"
  op: "NoOp"
  input: "^discriminator_predictions/dense/bias/Assign"
  input: "^discriminator_predictions/dense/bias/adam_m/Assign"
  input: "^discriminator_predictions/dense/bias/adam_v/Assign"
  input: "^discriminator_predictions/dense/kernel/Assign"
  input: "^discriminator_predictions/dense/kernel/adam_m/Assign"
  input: "^discriminator_predictions/dense/kernel/adam_v/Assign"
  input: "^discriminator_predictions/dense_1/bias/Assign"
  input: "^discriminator_predictions/dense_1/bias/adam_m/Assign"
  input: "^discriminator_predictions/dense_1/bias/adam_v/Assign"
  input: "^discriminator_predictions/dense_1/kernel/Assign"
  input: "^discriminator_predictions/dense_1/kernel/adam_m/Assign"
  input: "^discriminator_predictions/dense_1/kernel/adam_v/Assign"
  input: "^electra/embeddings/position_embeddings/Assign"
  input: "^electra/embeddings/position_embeddings/adam_m/Assign"
  input: "^electra/embeddings/position_embeddings/adam_v/Assign"
  input: "^electra/embeddings/token_type_embeddings/Assign"
  input: "^electra/embeddings/token_type_embeddings/adam_m/Assign"
  input: "^electra/embeddings/token_type_embeddings/adam_v/Assign"
  input: "^electra/embeddings/word_embeddings/Assign"
  input: "^electra/embeddings/word_embeddings/adam_m/Assign"
  input: "^electra/embeddings/word_embeddings/adam_v/Assign"
  input: "^electra/embeddings_1/LayerNorm/beta/Assign"
  input: "^electra/embeddings_1/LayerNorm/beta/adam_m/Assign"
  input: "^electra/embeddings_1/LayerNorm/beta/adam_v/Assign"
  input: "^electra/embeddings_1/LayerNorm/gamma/Assign"
  input: "^electra/embeddings_1/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/embeddings_1/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/embeddings_3/LayerNorm/beta/Assign"
  input: "^electra/embeddings_3/LayerNorm/beta/adam_m/Assign"
  input: "^electra/embeddings_3/LayerNorm/beta/adam_v/Assign"
  input: "^electra/embeddings_3/LayerNorm/gamma/Assign"
  input: "^electra/embeddings_3/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/embeddings_3/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_0/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_0/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_0/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_0/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_0/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_0/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_0/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_0/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_0/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_0/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_0/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_0/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_0/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_0/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_0/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_0/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_0/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_0/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_0/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_0/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_0/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_0/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_0/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_0/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_0/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_0/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_0/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_0/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_0/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_0/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_0/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_0/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_0/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_0/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_0/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_0/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_0/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_0/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_0/output/dense/bias/Assign"
  input: "^electra/encoder/layer_0/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_0/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_0/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_0/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_0/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_1/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_1/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_1/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_1/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_1/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_1/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_1/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_1/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_1/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_1/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_1/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_1/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_1/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_1/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_1/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_1/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_1/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_1/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_1/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_1/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_1/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_1/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_1/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_1/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_1/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_1/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_1/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_1/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_1/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_1/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_1/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_1/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_1/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_1/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_1/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_1/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_1/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_1/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_1/output/dense/bias/Assign"
  input: "^electra/encoder/layer_1/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_1/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_1/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_1/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_1/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_10/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_10/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_10/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_10/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_10/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_10/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_10/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_10/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_10/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_10/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_10/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_10/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_10/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_10/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_10/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_10/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_10/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_10/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_10/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_10/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_10/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_10/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_10/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_10/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_10/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_10/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_10/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_10/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_10/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_10/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_10/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_10/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_10/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_10/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_10/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_10/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_10/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_10/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_10/output/dense/bias/Assign"
  input: "^electra/encoder/layer_10/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_10/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_10/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_10/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_10/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_11/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_11/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_11/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_11/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_11/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_11/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_11/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_11/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_11/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_11/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_11/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_11/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_11/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_11/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_11/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_11/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_11/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_11/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_11/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_11/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_11/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_11/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_11/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_11/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_11/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_11/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_11/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_11/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_11/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_11/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_11/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_11/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_11/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_11/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_11/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_11/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_11/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_11/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_11/output/dense/bias/Assign"
  input: "^electra/encoder/layer_11/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_11/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_11/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_11/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_11/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_2/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_2/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_2/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_2/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_2/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_2/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_2/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_2/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_2/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_2/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_2/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_2/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_2/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_2/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_2/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_2/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_2/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_2/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_2/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_2/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_2/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_2/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_2/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_2/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_2/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_2/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_2/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_2/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_2/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_2/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_2/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_2/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_2/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_2/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_2/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_2/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_2/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_2/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_2/output/dense/bias/Assign"
  input: "^electra/encoder/layer_2/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_2/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_2/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_2/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_2/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_3/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_3/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_3/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_3/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_3/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_3/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_3/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_3/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_3/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_3/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_3/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_3/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_3/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_3/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_3/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_3/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_3/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_3/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_3/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_3/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_3/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_3/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_3/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_3/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_3/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_3/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_3/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_3/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_3/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_3/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_3/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_3/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_3/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_3/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_3/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_3/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_3/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_3/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_3/output/dense/bias/Assign"
  input: "^electra/encoder/layer_3/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_3/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_3/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_3/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_3/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_4/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_4/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_4/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_4/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_4/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_4/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_4/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_4/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_4/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_4/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_4/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_4/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_4/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_4/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_4/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_4/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_4/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_4/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_4/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_4/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_4/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_4/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_4/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_4/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_4/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_4/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_4/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_4/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_4/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_4/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_4/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_4/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_4/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_4/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_4/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_4/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_4/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_4/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_4/output/dense/bias/Assign"
  input: "^electra/encoder/layer_4/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_4/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_4/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_4/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_4/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_5/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_5/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_5/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_5/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_5/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_5/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_5/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_5/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_5/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_5/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_5/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_5/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_5/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_5/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_5/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_5/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_5/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_5/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_5/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_5/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_5/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_5/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_5/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_5/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_5/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_5/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_5/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_5/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_5/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_5/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_5/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_5/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_5/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_5/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_5/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_5/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_5/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_5/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_5/output/dense/bias/Assign"
  input: "^electra/encoder/layer_5/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_5/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_5/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_5/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_5/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_6/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_6/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_6/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_6/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_6/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_6/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_6/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_6/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_6/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_6/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_6/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_6/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_6/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_6/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_6/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_6/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_6/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_6/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_6/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_6/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_6/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_6/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_6/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_6/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_6/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_6/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_6/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_6/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_6/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_6/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_6/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_6/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_6/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_6/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_6/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_6/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_6/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_6/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_6/output/dense/bias/Assign"
  input: "^electra/encoder/layer_6/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_6/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_6/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_6/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_6/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_7/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_7/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_7/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_7/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_7/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_7/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_7/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_7/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_7/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_7/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_7/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_7/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_7/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_7/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_7/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_7/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_7/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_7/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_7/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_7/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_7/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_7/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_7/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_7/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_7/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_7/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_7/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_7/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_7/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_7/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_7/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_7/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_7/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_7/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_7/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_7/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_7/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_7/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_7/output/dense/bias/Assign"
  input: "^electra/encoder/layer_7/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_7/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_7/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_7/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_7/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_8/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_8/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_8/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_8/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_8/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_8/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_8/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_8/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_8/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_8/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_8/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_8/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_8/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_8/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_8/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_8/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_8/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_8/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_8/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_8/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_8/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_8/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_8/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_8/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_8/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_8/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_8/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_8/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_8/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_8/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_8/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_8/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_8/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_8/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_8/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_8/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_8/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_8/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_8/output/dense/bias/Assign"
  input: "^electra/encoder/layer_8/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_8/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_8/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_8/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_8/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_9/attention/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_9/attention/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_9/attention/output/dense/bias/Assign"
  input: "^electra/encoder/layer_9/attention/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_9/attention/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_9/attention/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_9/attention/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_9/attention/output/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_9/attention/self/key/bias/Assign"
  input: "^electra/encoder/layer_9/attention/self/key/bias/adam_m/Assign"
  input: "^electra/encoder/layer_9/attention/self/key/bias/adam_v/Assign"
  input: "^electra/encoder/layer_9/attention/self/key/kernel/Assign"
  input: "^electra/encoder/layer_9/attention/self/key/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_9/attention/self/key/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_9/attention/self/query/bias/Assign"
  input: "^electra/encoder/layer_9/attention/self/query/bias/adam_m/Assign"
  input: "^electra/encoder/layer_9/attention/self/query/bias/adam_v/Assign"
  input: "^electra/encoder/layer_9/attention/self/query/kernel/Assign"
  input: "^electra/encoder/layer_9/attention/self/query/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_9/attention/self/query/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_9/attention/self/value/bias/Assign"
  input: "^electra/encoder/layer_9/attention/self/value/bias/adam_m/Assign"
  input: "^electra/encoder/layer_9/attention/self/value/bias/adam_v/Assign"
  input: "^electra/encoder/layer_9/attention/self/value/kernel/Assign"
  input: "^electra/encoder/layer_9/attention/self/value/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_9/attention/self/value/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_9/intermediate/dense/bias/Assign"
  input: "^electra/encoder/layer_9/intermediate/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_9/intermediate/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_9/intermediate/dense/kernel/Assign"
  input: "^electra/encoder/layer_9/intermediate/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_9/intermediate/dense/kernel/adam_v/Assign"
  input: "^electra/encoder/layer_9/output/LayerNorm/beta/Assign"
  input: "^electra/encoder/layer_9/output/LayerNorm/beta/adam_m/Assign"
  input: "^electra/encoder/layer_9/output/LayerNorm/beta/adam_v/Assign"
  input: "^electra/encoder/layer_9/output/LayerNorm/gamma/Assign"
  input: "^electra/encoder/layer_9/output/LayerNorm/gamma/adam_m/Assign"
  input: "^electra/encoder/layer_9/output/LayerNorm/gamma/adam_v/Assign"
  input: "^electra/encoder/layer_9/output/dense/bias/Assign"
  input: "^electra/encoder/layer_9/output/dense/bias/adam_m/Assign"
  input: "^electra/encoder/layer_9/output/dense/bias/adam_v/Assign"
  input: "^electra/encoder/layer_9/output/dense/kernel/Assign"
  input: "^electra/encoder/layer_9/output/dense/kernel/adam_m/Assign"
  input: "^electra/encoder/layer_9/output/dense/kernel/adam_v/Assign"
  input: "^generator/embeddings_project/bias/Assign"
  input: "^generator/embeddings_project/bias/adam_m/Assign"
  input: "^generator/embeddings_project/bias/adam_v/Assign"
  input: "^generator/embeddings_project/kernel/Assign"
  input: "^generator/embeddings_project/kernel/adam_m/Assign"
  input: "^generator/embeddings_project/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_0/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_0/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_0/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_0/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_0/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_0/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_0/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_0/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_0/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_0/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_0/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_0/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_0/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_0/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_0/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_0/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_0/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_0/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_0/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_0/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_0/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_0/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_0/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_0/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_0/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_0/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_0/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_0/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_0/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_0/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_0/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_0/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_0/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_0/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_0/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_0/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_0/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_0/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_0/output/dense/bias/Assign"
  input: "^generator/encoder/layer_0/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_0/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_0/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_0/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_0/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_1/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_1/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_1/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_1/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_1/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_1/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_1/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_1/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_1/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_1/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_1/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_1/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_1/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_1/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_1/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_1/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_1/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_1/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_1/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_1/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_1/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_1/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_1/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_1/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_1/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_1/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_1/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_1/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_1/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_1/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_1/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_1/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_1/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_1/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_1/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_1/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_1/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_1/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_1/output/dense/bias/Assign"
  input: "^generator/encoder/layer_1/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_1/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_1/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_1/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_1/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_10/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_10/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_10/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_10/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_10/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_10/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_10/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_10/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_10/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_10/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_10/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_10/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_10/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_10/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_10/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_10/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_10/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_10/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_10/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_10/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_10/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_10/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_10/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_10/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_10/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_10/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_10/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_10/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_10/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_10/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_10/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_10/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_10/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_10/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_10/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_10/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_10/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_10/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_10/output/dense/bias/Assign"
  input: "^generator/encoder/layer_10/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_10/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_10/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_10/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_10/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_11/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_11/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_11/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_11/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_11/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_11/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_11/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_11/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_11/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_11/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_11/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_11/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_11/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_11/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_11/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_11/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_11/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_11/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_11/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_11/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_11/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_11/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_11/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_11/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_11/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_11/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_11/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_11/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_11/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_11/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_11/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_11/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_11/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_11/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_11/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_11/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_11/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_11/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_11/output/dense/bias/Assign"
  input: "^generator/encoder/layer_11/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_11/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_11/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_11/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_11/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_2/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_2/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_2/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_2/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_2/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_2/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_2/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_2/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_2/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_2/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_2/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_2/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_2/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_2/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_2/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_2/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_2/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_2/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_2/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_2/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_2/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_2/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_2/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_2/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_2/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_2/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_2/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_2/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_2/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_2/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_2/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_2/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_2/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_2/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_2/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_2/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_2/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_2/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_2/output/dense/bias/Assign"
  input: "^generator/encoder/layer_2/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_2/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_2/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_2/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_2/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_3/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_3/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_3/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_3/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_3/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_3/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_3/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_3/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_3/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_3/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_3/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_3/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_3/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_3/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_3/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_3/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_3/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_3/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_3/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_3/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_3/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_3/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_3/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_3/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_3/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_3/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_3/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_3/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_3/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_3/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_3/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_3/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_3/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_3/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_3/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_3/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_3/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_3/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_3/output/dense/bias/Assign"
  input: "^generator/encoder/layer_3/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_3/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_3/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_3/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_3/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_4/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_4/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_4/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_4/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_4/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_4/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_4/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_4/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_4/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_4/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_4/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_4/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_4/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_4/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_4/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_4/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_4/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_4/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_4/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_4/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_4/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_4/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_4/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_4/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_4/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_4/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_4/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_4/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_4/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_4/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_4/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_4/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_4/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_4/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_4/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_4/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_4/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_4/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_4/output/dense/bias/Assign"
  input: "^generator/encoder/layer_4/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_4/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_4/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_4/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_4/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_5/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_5/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_5/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_5/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_5/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_5/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_5/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_5/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_5/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_5/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_5/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_5/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_5/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_5/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_5/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_5/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_5/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_5/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_5/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_5/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_5/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_5/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_5/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_5/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_5/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_5/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_5/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_5/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_5/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_5/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_5/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_5/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_5/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_5/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_5/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_5/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_5/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_5/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_5/output/dense/bias/Assign"
  input: "^generator/encoder/layer_5/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_5/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_5/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_5/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_5/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_6/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_6/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_6/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_6/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_6/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_6/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_6/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_6/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_6/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_6/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_6/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_6/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_6/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_6/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_6/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_6/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_6/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_6/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_6/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_6/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_6/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_6/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_6/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_6/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_6/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_6/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_6/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_6/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_6/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_6/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_6/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_6/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_6/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_6/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_6/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_6/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_6/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_6/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_6/output/dense/bias/Assign"
  input: "^generator/encoder/layer_6/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_6/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_6/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_6/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_6/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_7/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_7/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_7/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_7/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_7/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_7/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_7/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_7/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_7/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_7/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_7/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_7/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_7/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_7/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_7/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_7/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_7/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_7/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_7/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_7/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_7/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_7/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_7/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_7/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_7/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_7/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_7/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_7/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_7/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_7/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_7/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_7/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_7/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_7/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_7/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_7/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_7/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_7/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_7/output/dense/bias/Assign"
  input: "^generator/encoder/layer_7/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_7/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_7/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_7/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_7/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_8/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_8/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_8/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_8/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_8/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_8/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_8/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_8/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_8/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_8/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_8/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_8/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_8/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_8/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_8/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_8/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_8/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_8/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_8/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_8/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_8/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_8/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_8/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_8/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_8/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_8/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_8/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_8/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_8/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_8/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_8/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_8/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_8/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_8/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_8/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_8/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_8/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_8/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_8/output/dense/bias/Assign"
  input: "^generator/encoder/layer_8/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_8/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_8/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_8/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_8/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_9/attention/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_9/attention/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_9/attention/output/dense/bias/Assign"
  input: "^generator/encoder/layer_9/attention/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_9/attention/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_9/attention/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_9/attention/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_9/attention/output/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_9/attention/self/key/bias/Assign"
  input: "^generator/encoder/layer_9/attention/self/key/bias/adam_m/Assign"
  input: "^generator/encoder/layer_9/attention/self/key/bias/adam_v/Assign"
  input: "^generator/encoder/layer_9/attention/self/key/kernel/Assign"
  input: "^generator/encoder/layer_9/attention/self/key/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_9/attention/self/key/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_9/attention/self/query/bias/Assign"
  input: "^generator/encoder/layer_9/attention/self/query/bias/adam_m/Assign"
  input: "^generator/encoder/layer_9/attention/self/query/bias/adam_v/Assign"
  input: "^generator/encoder/layer_9/attention/self/query/kernel/Assign"
  input: "^generator/encoder/layer_9/attention/self/query/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_9/attention/self/query/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_9/attention/self/value/bias/Assign"
  input: "^generator/encoder/layer_9/attention/self/value/bias/adam_m/Assign"
  input: "^generator/encoder/layer_9/attention/self/value/bias/adam_v/Assign"
  input: "^generator/encoder/layer_9/attention/self/value/kernel/Assign"
  input: "^generator/encoder/layer_9/attention/self/value/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_9/attention/self/value/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_9/intermediate/dense/bias/Assign"
  input: "^generator/encoder/layer_9/intermediate/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_9/intermediate/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_9/intermediate/dense/kernel/Assign"
  input: "^generator/encoder/layer_9/intermediate/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_9/intermediate/dense/kernel/adam_v/Assign"
  input: "^generator/encoder/layer_9/output/LayerNorm/beta/Assign"
  input: "^generator/encoder/layer_9/output/LayerNorm/beta/adam_m/Assign"
  input: "^generator/encoder/layer_9/output/LayerNorm/beta/adam_v/Assign"
  input: "^generator/encoder/layer_9/output/LayerNorm/gamma/Assign"
  input: "^generator/encoder/layer_9/output/LayerNorm/gamma/adam_m/Assign"
  input: "^generator/encoder/layer_9/output/LayerNorm/gamma/adam_v/Assign"
  input: "^generator/encoder/layer_9/output/dense/bias/Assign"
  input: "^generator/encoder/layer_9/output/dense/bias/adam_m/Assign"
  input: "^generator/encoder/layer_9/output/dense/bias/adam_v/Assign"
  input: "^generator/encoder/layer_9/output/dense/kernel/Assign"
  input: "^generator/encoder/layer_9/output/dense/kernel/adam_m/Assign"
  input: "^generator/encoder/layer_9/output/dense/kernel/adam_v/Assign"
  input: "^generator_predictions/LayerNorm/beta/Assign"
  input: "^generator_predictions/LayerNorm/beta/adam_m/Assign"
  input: "^generator_predictions/LayerNorm/beta/adam_v/Assign"
  input: "^generator_predictions/LayerNorm/gamma/Assign"
  input: "^generator_predictions/LayerNorm/gamma/adam_m/Assign"
  input: "^generator_predictions/LayerNorm/gamma/adam_v/Assign"
  input: "^generator_predictions/dense/bias/Assign"
  input: "^generator_predictions/dense/bias/adam_m/Assign"
  input: "^generator_predictions/dense/bias/adam_v/Assign"
  input: "^generator_predictions/dense/kernel/Assign"
  input: "^generator_predictions/dense/kernel/adam_m/Assign"
  input: "^generator_predictions/dense/kernel/adam_v/Assign"
  input: "^generator_predictions/output_bias/Assign"
  input: "^generator_predictions/output_bias/adam_m/Assign"
  input: "^generator_predictions/output_bias/adam_v/Assign"
  input: "^global_step/Assign"
}
node {
  name: "init_1"
  op: "NoOp"
}
node {
  name: "group_deps_4"
  op: "NoOp"
  input: "^init"
  input: "^init_1"
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "global_step"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/word_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_2"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/token_type_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_3"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/position_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_4"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_5"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_6"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_7"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_8"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_9"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_10"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_11"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_12"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_13"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_14"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_15"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_16"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_17"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_18"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_19"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_20"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_21"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_22"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_23"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_24"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_25"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_26"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_27"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_28"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_29"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_30"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_31"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_32"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_33"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_34"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_35"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_36"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_37"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_38"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_39"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_40"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_41"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_42"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_43"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_44"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_45"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_46"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_47"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_48"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_49"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_50"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_51"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_52"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_53"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_54"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_55"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_56"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_57"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_58"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_59"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_60"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_61"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_62"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_63"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_64"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_65"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_66"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_67"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_68"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_69"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_70"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_71"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_72"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_73"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_74"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_75"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_76"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_77"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_78"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_79"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_80"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_81"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_82"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_83"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_84"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_85"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_86"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_87"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_88"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_89"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_90"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_91"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_92"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_93"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_94"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_95"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_96"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_97"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_98"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_99"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_100"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_101"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_102"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_103"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_104"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_105"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_106"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_107"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_108"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_109"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_110"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_111"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_112"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_113"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_114"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_115"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_116"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_117"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_118"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_119"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_120"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_121"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_122"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_123"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_124"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_125"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_126"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_127"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_128"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_129"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_130"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_131"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_132"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_133"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_134"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_135"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_136"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_137"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_138"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_139"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_140"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_141"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_142"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_143"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_144"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_145"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_146"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_147"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_148"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_149"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_150"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_151"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_152"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_153"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_154"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_155"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_156"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_157"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_158"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_159"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_160"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_161"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_162"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_163"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_164"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_165"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_166"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_167"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_168"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_169"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_170"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_171"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_172"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_173"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_174"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_175"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_176"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_177"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_178"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_179"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_180"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_181"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_182"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_183"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_184"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_185"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_186"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_187"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_188"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_189"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_190"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_191"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_192"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_193"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_194"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_195"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_196"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_197"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_198"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_199"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_200"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_201"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_202"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_203"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_204"
  op: "VarIsInitializedOp"
  input: "generator_predictions/output_bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_205"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_206"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_207"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_208"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_209"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_210"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_211"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_212"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_213"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_214"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_215"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_216"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_217"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_218"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_219"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_220"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_221"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_222"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_223"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_224"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_225"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_226"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_227"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_228"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_229"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_230"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_231"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_232"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_233"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_234"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_235"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_236"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_237"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_238"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_239"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_240"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_241"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_242"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_243"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_244"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_245"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_246"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_247"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_248"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_249"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_250"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_251"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_252"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_253"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_254"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_255"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_256"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_257"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_258"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_259"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_260"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_261"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_262"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_263"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_264"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_265"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_266"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_267"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_268"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_269"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_270"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_271"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_272"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_273"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_274"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_275"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_276"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_277"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_278"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_279"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_280"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_281"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_282"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_283"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_284"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_285"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_286"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_287"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_288"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_289"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_290"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_291"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_292"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_293"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_294"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_295"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_296"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_297"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_298"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_299"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_300"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_301"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_302"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_303"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_304"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_305"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_306"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_307"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_308"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_309"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_310"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_311"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_312"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_313"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_314"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_315"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_316"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_317"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_318"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_319"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_320"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_321"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_322"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_323"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_324"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_325"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_326"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_327"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_328"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_329"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_330"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_331"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_332"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_333"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_334"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_335"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_336"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_337"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_338"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_339"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_340"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_341"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_342"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_343"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_344"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_345"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_346"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_347"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_348"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_349"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_350"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_351"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_352"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_353"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_354"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_355"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_356"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_357"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_358"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_359"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_360"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_361"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_362"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_363"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_364"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_365"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_366"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_367"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_368"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_369"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_370"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_371"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_372"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_373"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_374"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_375"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_376"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_377"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_378"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_379"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_380"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_381"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_382"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_383"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_384"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_385"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_386"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_387"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_388"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_389"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_390"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_391"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_392"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_393"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_394"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_395"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_396"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_397"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_398"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_399"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_400"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_401"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_402"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_403"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/word_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_404"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/word_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_405"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/token_type_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_406"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/token_type_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_407"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/position_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_408"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/position_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_409"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_410"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_411"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_412"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_413"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_414"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_415"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_416"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_417"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_418"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_419"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_420"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_421"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_422"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_423"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_424"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_425"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_426"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_427"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_428"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_429"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_430"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_431"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_432"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_433"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_434"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_435"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_436"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_437"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_438"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_439"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_440"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_441"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_442"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_443"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_444"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_445"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_446"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_447"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_448"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_449"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_450"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_451"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_452"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_453"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_454"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_455"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_456"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_457"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_458"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_459"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_460"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_461"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_462"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_463"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_464"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_465"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_466"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_467"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_468"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_469"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_470"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_471"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_472"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_473"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_474"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_475"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_476"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_477"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_478"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_479"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_480"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_481"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_482"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_483"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_484"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_485"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_486"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_487"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_488"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_489"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_490"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_491"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_492"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_493"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_494"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_495"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_496"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_497"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_498"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_499"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_500"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_501"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_502"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_503"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_504"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_505"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_506"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_507"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_508"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_509"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_510"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_511"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_512"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_513"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_514"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_515"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_516"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_517"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_518"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_519"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_520"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_521"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_522"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_523"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_524"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_525"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_526"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_527"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_528"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_529"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_530"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_531"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_532"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_533"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_534"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_535"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_536"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_537"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_538"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_539"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_540"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_541"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_542"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_543"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_544"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_545"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_546"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_547"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_548"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_549"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_550"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_551"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_552"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_553"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_554"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_555"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_556"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_557"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_558"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_559"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_560"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_561"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_562"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_563"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_564"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_565"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_566"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_567"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_568"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_569"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_570"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_571"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_572"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_573"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_574"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_575"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_576"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_577"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_578"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_579"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_580"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_581"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_582"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_583"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_584"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_585"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_586"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_587"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_588"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_589"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_590"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_591"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_592"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_593"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_594"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_595"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_596"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_597"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_598"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_599"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_600"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_601"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_602"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_603"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_604"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_605"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_606"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_607"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_608"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_609"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_610"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_611"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_612"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_613"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_614"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_615"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_616"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_617"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_618"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_619"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_620"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_621"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_622"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_623"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_624"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_625"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_626"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_627"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_628"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_629"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_630"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_631"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_632"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_633"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_634"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_635"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_636"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_637"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_638"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_639"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_640"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_641"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_642"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_643"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_644"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_645"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_646"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_647"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_648"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_649"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_650"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_651"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_652"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_653"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_654"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_655"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_656"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_657"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_658"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_659"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_660"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_661"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_662"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_663"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_664"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_665"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_666"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_667"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_668"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_669"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_670"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_671"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_672"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_673"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_674"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_675"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_676"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_677"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_678"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_679"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_680"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_681"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_682"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_683"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_684"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_685"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_686"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_687"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_688"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_689"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_690"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_691"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_692"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_693"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_694"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_695"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_696"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_697"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_698"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_699"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_700"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_701"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_702"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_703"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_704"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_705"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_706"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_707"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_708"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_709"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_710"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_711"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_712"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_713"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_714"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_715"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_716"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_717"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_718"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_719"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_720"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_721"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_722"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_723"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_724"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_725"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_726"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_727"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_728"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_729"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_730"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_731"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_732"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_733"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_734"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_735"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_736"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_737"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_738"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_739"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_740"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_741"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_742"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_743"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_744"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_745"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_746"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_747"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_748"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_749"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_750"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_751"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_752"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_753"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_754"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_755"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_756"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_757"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_758"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_759"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_760"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_761"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_762"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_763"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_764"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_765"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_766"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_767"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_768"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_769"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_770"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_771"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_772"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_773"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_774"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_775"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_776"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_777"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_778"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_779"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_780"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_781"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_782"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_783"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_784"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_785"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_786"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_787"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_788"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_789"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_790"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_791"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_792"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_793"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_794"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_795"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_796"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_797"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_798"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_799"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_800"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_801"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_802"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_803"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_804"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_805"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_806"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_807"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_808"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_809"
  op: "VarIsInitializedOp"
  input: "generator_predictions/output_bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_810"
  op: "VarIsInitializedOp"
  input: "generator_predictions/output_bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_811"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_812"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_813"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_814"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_815"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_816"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_817"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_818"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_819"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_820"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_821"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_822"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_823"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_824"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_825"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_826"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_827"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_828"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_829"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_830"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_831"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_832"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_833"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_834"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_835"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_836"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_837"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_838"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_839"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_840"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_841"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_842"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_843"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_844"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_845"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_846"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_847"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_848"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_849"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_850"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_851"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_852"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_853"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_854"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_855"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_856"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_857"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_858"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_859"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_860"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_861"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_862"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_863"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_864"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_865"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_866"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_867"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_868"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_869"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_870"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_871"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_872"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_873"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_874"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_875"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_876"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_877"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_878"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_879"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_880"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_881"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_882"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_883"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_884"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_885"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_886"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_887"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_888"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_889"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_890"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_891"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_892"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_893"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_894"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_895"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_896"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_897"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_898"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_899"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_900"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_901"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_902"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_903"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_904"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_905"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_906"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_907"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_908"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_909"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_910"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_911"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_912"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_913"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_914"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_915"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_916"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_917"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_918"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_919"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_920"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_921"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_922"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_923"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_924"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_925"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_926"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_927"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_928"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_929"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_930"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_931"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_932"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_933"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_934"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_935"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_936"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_937"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_938"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_939"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_940"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_941"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_942"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_943"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_944"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_945"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_946"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_947"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_948"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_949"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_950"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_951"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_952"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_953"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_954"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_955"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_956"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_957"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_958"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_959"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_960"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_961"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_962"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_963"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_964"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_965"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_966"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_967"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_968"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_969"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_970"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_971"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_972"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_973"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_974"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_975"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_976"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_977"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_978"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_979"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_980"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_981"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_982"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_983"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_984"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_985"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_986"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_987"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_988"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_989"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_990"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_991"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_992"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_993"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_994"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_995"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_996"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_997"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_998"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_999"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1000"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1001"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1002"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1003"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1004"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1005"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1006"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1007"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1008"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1009"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1010"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1011"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1012"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1013"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1014"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1015"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1016"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1017"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1018"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1019"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1020"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1021"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1022"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1023"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1024"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1025"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1026"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1027"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1028"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1029"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1030"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1031"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1032"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1033"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1034"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1035"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1036"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1037"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1038"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1039"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1040"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1041"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1042"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1043"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1044"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1045"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1046"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1047"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1048"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1049"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1050"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1051"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1052"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1053"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1054"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1055"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1056"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1057"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1058"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1059"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1060"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1061"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1062"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1063"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1064"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1065"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1066"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1067"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1068"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1069"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1070"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1071"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1072"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1073"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1074"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1075"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1076"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1077"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1078"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1079"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1080"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1081"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1082"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1083"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1084"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1085"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1086"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1087"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1088"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1089"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1090"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1091"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1092"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1093"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1094"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1095"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1096"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1097"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1098"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1099"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1100"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1101"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1102"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1103"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1104"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1105"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1106"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1107"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1108"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1109"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1110"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1111"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1112"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1113"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1114"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1115"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1116"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1117"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1118"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1119"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1120"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1121"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1122"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1123"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1124"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1125"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1126"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1127"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1128"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1129"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1130"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1131"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1132"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1133"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1134"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1135"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1136"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1137"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1138"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1139"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1140"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1141"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1142"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1143"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1144"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1145"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1146"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1147"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1148"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1149"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1150"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1151"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1152"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1153"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1154"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1155"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1156"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1157"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1158"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1159"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1160"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1161"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1162"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1163"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1164"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1165"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1166"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1167"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1168"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1169"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1170"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1171"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1172"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1173"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1174"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1175"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1176"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1177"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1178"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1179"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1180"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1181"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1182"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1183"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1184"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1185"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1186"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1187"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1188"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1189"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1190"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1191"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1192"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1193"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1194"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1195"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1196"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1197"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1198"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1199"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1200"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1201"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1202"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1203"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1204"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1205"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1206"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/VarIsInitializedOp_1207"
  op: "VarIsInitializedOp"
  input: "tpu_estimator/iterations_per_loop"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/stack"
  op: "Pack"
  input: "report_uninitialized_variables/VarIsInitializedOp"
  input: "report_uninitialized_variables/VarIsInitializedOp_1"
  input: "report_uninitialized_variables/VarIsInitializedOp_2"
  input: "report_uninitialized_variables/VarIsInitializedOp_3"
  input: "report_uninitialized_variables/VarIsInitializedOp_4"
  input: "report_uninitialized_variables/VarIsInitializedOp_5"
  input: "report_uninitialized_variables/VarIsInitializedOp_6"
  input: "report_uninitialized_variables/VarIsInitializedOp_7"
  input: "report_uninitialized_variables/VarIsInitializedOp_8"
  input: "report_uninitialized_variables/VarIsInitializedOp_9"
  input: "report_uninitialized_variables/VarIsInitializedOp_10"
  input: "report_uninitialized_variables/VarIsInitializedOp_11"
  input: "report_uninitialized_variables/VarIsInitializedOp_12"
  input: "report_uninitialized_variables/VarIsInitializedOp_13"
  input: "report_uninitialized_variables/VarIsInitializedOp_14"
  input: "report_uninitialized_variables/VarIsInitializedOp_15"
  input: "report_uninitialized_variables/VarIsInitializedOp_16"
  input: "report_uninitialized_variables/VarIsInitializedOp_17"
  input: "report_uninitialized_variables/VarIsInitializedOp_18"
  input: "report_uninitialized_variables/VarIsInitializedOp_19"
  input: "report_uninitialized_variables/VarIsInitializedOp_20"
  input: "report_uninitialized_variables/VarIsInitializedOp_21"
  input: "report_uninitialized_variables/VarIsInitializedOp_22"
  input: "report_uninitialized_variables/VarIsInitializedOp_23"
  input: "report_uninitialized_variables/VarIsInitializedOp_24"
  input: "report_uninitialized_variables/VarIsInitializedOp_25"
  input: "report_uninitialized_variables/VarIsInitializedOp_26"
  input: "report_uninitialized_variables/VarIsInitializedOp_27"
  input: "report_uninitialized_variables/VarIsInitializedOp_28"
  input: "report_uninitialized_variables/VarIsInitializedOp_29"
  input: "report_uninitialized_variables/VarIsInitializedOp_30"
  input: "report_uninitialized_variables/VarIsInitializedOp_31"
  input: "report_uninitialized_variables/VarIsInitializedOp_32"
  input: "report_uninitialized_variables/VarIsInitializedOp_33"
  input: "report_uninitialized_variables/VarIsInitializedOp_34"
  input: "report_uninitialized_variables/VarIsInitializedOp_35"
  input: "report_uninitialized_variables/VarIsInitializedOp_36"
  input: "report_uninitialized_variables/VarIsInitializedOp_37"
  input: "report_uninitialized_variables/VarIsInitializedOp_38"
  input: "report_uninitialized_variables/VarIsInitializedOp_39"
  input: "report_uninitialized_variables/VarIsInitializedOp_40"
  input: "report_uninitialized_variables/VarIsInitializedOp_41"
  input: "report_uninitialized_variables/VarIsInitializedOp_42"
  input: "report_uninitialized_variables/VarIsInitializedOp_43"
  input: "report_uninitialized_variables/VarIsInitializedOp_44"
  input: "report_uninitialized_variables/VarIsInitializedOp_45"
  input: "report_uninitialized_variables/VarIsInitializedOp_46"
  input: "report_uninitialized_variables/VarIsInitializedOp_47"
  input: "report_uninitialized_variables/VarIsInitializedOp_48"
  input: "report_uninitialized_variables/VarIsInitializedOp_49"
  input: "report_uninitialized_variables/VarIsInitializedOp_50"
  input: "report_uninitialized_variables/VarIsInitializedOp_51"
  input: "report_uninitialized_variables/VarIsInitializedOp_52"
  input: "report_uninitialized_variables/VarIsInitializedOp_53"
  input: "report_uninitialized_variables/VarIsInitializedOp_54"
  input: "report_uninitialized_variables/VarIsInitializedOp_55"
  input: "report_uninitialized_variables/VarIsInitializedOp_56"
  input: "report_uninitialized_variables/VarIsInitializedOp_57"
  input: "report_uninitialized_variables/VarIsInitializedOp_58"
  input: "report_uninitialized_variables/VarIsInitializedOp_59"
  input: "report_uninitialized_variables/VarIsInitializedOp_60"
  input: "report_uninitialized_variables/VarIsInitializedOp_61"
  input: "report_uninitialized_variables/VarIsInitializedOp_62"
  input: "report_uninitialized_variables/VarIsInitializedOp_63"
  input: "report_uninitialized_variables/VarIsInitializedOp_64"
  input: "report_uninitialized_variables/VarIsInitializedOp_65"
  input: "report_uninitialized_variables/VarIsInitializedOp_66"
  input: "report_uninitialized_variables/VarIsInitializedOp_67"
  input: "report_uninitialized_variables/VarIsInitializedOp_68"
  input: "report_uninitialized_variables/VarIsInitializedOp_69"
  input: "report_uninitialized_variables/VarIsInitializedOp_70"
  input: "report_uninitialized_variables/VarIsInitializedOp_71"
  input: "report_uninitialized_variables/VarIsInitializedOp_72"
  input: "report_uninitialized_variables/VarIsInitializedOp_73"
  input: "report_uninitialized_variables/VarIsInitializedOp_74"
  input: "report_uninitialized_variables/VarIsInitializedOp_75"
  input: "report_uninitialized_variables/VarIsInitializedOp_76"
  input: "report_uninitialized_variables/VarIsInitializedOp_77"
  input: "report_uninitialized_variables/VarIsInitializedOp_78"
  input: "report_uninitialized_variables/VarIsInitializedOp_79"
  input: "report_uninitialized_variables/VarIsInitializedOp_80"
  input: "report_uninitialized_variables/VarIsInitializedOp_81"
  input: "report_uninitialized_variables/VarIsInitializedOp_82"
  input: "report_uninitialized_variables/VarIsInitializedOp_83"
  input: "report_uninitialized_variables/VarIsInitializedOp_84"
  input: "report_uninitialized_variables/VarIsInitializedOp_85"
  input: "report_uninitialized_variables/VarIsInitializedOp_86"
  input: "report_uninitialized_variables/VarIsInitializedOp_87"
  input: "report_uninitialized_variables/VarIsInitializedOp_88"
  input: "report_uninitialized_variables/VarIsInitializedOp_89"
  input: "report_uninitialized_variables/VarIsInitializedOp_90"
  input: "report_uninitialized_variables/VarIsInitializedOp_91"
  input: "report_uninitialized_variables/VarIsInitializedOp_92"
  input: "report_uninitialized_variables/VarIsInitializedOp_93"
  input: "report_uninitialized_variables/VarIsInitializedOp_94"
  input: "report_uninitialized_variables/VarIsInitializedOp_95"
  input: "report_uninitialized_variables/VarIsInitializedOp_96"
  input: "report_uninitialized_variables/VarIsInitializedOp_97"
  input: "report_uninitialized_variables/VarIsInitializedOp_98"
  input: "report_uninitialized_variables/VarIsInitializedOp_99"
  input: "report_uninitialized_variables/VarIsInitializedOp_100"
  input: "report_uninitialized_variables/VarIsInitializedOp_101"
  input: "report_uninitialized_variables/VarIsInitializedOp_102"
  input: "report_uninitialized_variables/VarIsInitializedOp_103"
  input: "report_uninitialized_variables/VarIsInitializedOp_104"
  input: "report_uninitialized_variables/VarIsInitializedOp_105"
  input: "report_uninitialized_variables/VarIsInitializedOp_106"
  input: "report_uninitialized_variables/VarIsInitializedOp_107"
  input: "report_uninitialized_variables/VarIsInitializedOp_108"
  input: "report_uninitialized_variables/VarIsInitializedOp_109"
  input: "report_uninitialized_variables/VarIsInitializedOp_110"
  input: "report_uninitialized_variables/VarIsInitializedOp_111"
  input: "report_uninitialized_variables/VarIsInitializedOp_112"
  input: "report_uninitialized_variables/VarIsInitializedOp_113"
  input: "report_uninitialized_variables/VarIsInitializedOp_114"
  input: "report_uninitialized_variables/VarIsInitializedOp_115"
  input: "report_uninitialized_variables/VarIsInitializedOp_116"
  input: "report_uninitialized_variables/VarIsInitializedOp_117"
  input: "report_uninitialized_variables/VarIsInitializedOp_118"
  input: "report_uninitialized_variables/VarIsInitializedOp_119"
  input: "report_uninitialized_variables/VarIsInitializedOp_120"
  input: "report_uninitialized_variables/VarIsInitializedOp_121"
  input: "report_uninitialized_variables/VarIsInitializedOp_122"
  input: "report_uninitialized_variables/VarIsInitializedOp_123"
  input: "report_uninitialized_variables/VarIsInitializedOp_124"
  input: "report_uninitialized_variables/VarIsInitializedOp_125"
  input: "report_uninitialized_variables/VarIsInitializedOp_126"
  input: "report_uninitialized_variables/VarIsInitializedOp_127"
  input: "report_uninitialized_variables/VarIsInitializedOp_128"
  input: "report_uninitialized_variables/VarIsInitializedOp_129"
  input: "report_uninitialized_variables/VarIsInitializedOp_130"
  input: "report_uninitialized_variables/VarIsInitializedOp_131"
  input: "report_uninitialized_variables/VarIsInitializedOp_132"
  input: "report_uninitialized_variables/VarIsInitializedOp_133"
  input: "report_uninitialized_variables/VarIsInitializedOp_134"
  input: "report_uninitialized_variables/VarIsInitializedOp_135"
  input: "report_uninitialized_variables/VarIsInitializedOp_136"
  input: "report_uninitialized_variables/VarIsInitializedOp_137"
  input: "report_uninitialized_variables/VarIsInitializedOp_138"
  input: "report_uninitialized_variables/VarIsInitializedOp_139"
  input: "report_uninitialized_variables/VarIsInitializedOp_140"
  input: "report_uninitialized_variables/VarIsInitializedOp_141"
  input: "report_uninitialized_variables/VarIsInitializedOp_142"
  input: "report_uninitialized_variables/VarIsInitializedOp_143"
  input: "report_uninitialized_variables/VarIsInitializedOp_144"
  input: "report_uninitialized_variables/VarIsInitializedOp_145"
  input: "report_uninitialized_variables/VarIsInitializedOp_146"
  input: "report_uninitialized_variables/VarIsInitializedOp_147"
  input: "report_uninitialized_variables/VarIsInitializedOp_148"
  input: "report_uninitialized_variables/VarIsInitializedOp_149"
  input: "report_uninitialized_variables/VarIsInitializedOp_150"
  input: "report_uninitialized_variables/VarIsInitializedOp_151"
  input: "report_uninitialized_variables/VarIsInitializedOp_152"
  input: "report_uninitialized_variables/VarIsInitializedOp_153"
  input: "report_uninitialized_variables/VarIsInitializedOp_154"
  input: "report_uninitialized_variables/VarIsInitializedOp_155"
  input: "report_uninitialized_variables/VarIsInitializedOp_156"
  input: "report_uninitialized_variables/VarIsInitializedOp_157"
  input: "report_uninitialized_variables/VarIsInitializedOp_158"
  input: "report_uninitialized_variables/VarIsInitializedOp_159"
  input: "report_uninitialized_variables/VarIsInitializedOp_160"
  input: "report_uninitialized_variables/VarIsInitializedOp_161"
  input: "report_uninitialized_variables/VarIsInitializedOp_162"
  input: "report_uninitialized_variables/VarIsInitializedOp_163"
  input: "report_uninitialized_variables/VarIsInitializedOp_164"
  input: "report_uninitialized_variables/VarIsInitializedOp_165"
  input: "report_uninitialized_variables/VarIsInitializedOp_166"
  input: "report_uninitialized_variables/VarIsInitializedOp_167"
  input: "report_uninitialized_variables/VarIsInitializedOp_168"
  input: "report_uninitialized_variables/VarIsInitializedOp_169"
  input: "report_uninitialized_variables/VarIsInitializedOp_170"
  input: "report_uninitialized_variables/VarIsInitializedOp_171"
  input: "report_uninitialized_variables/VarIsInitializedOp_172"
  input: "report_uninitialized_variables/VarIsInitializedOp_173"
  input: "report_uninitialized_variables/VarIsInitializedOp_174"
  input: "report_uninitialized_variables/VarIsInitializedOp_175"
  input: "report_uninitialized_variables/VarIsInitializedOp_176"
  input: "report_uninitialized_variables/VarIsInitializedOp_177"
  input: "report_uninitialized_variables/VarIsInitializedOp_178"
  input: "report_uninitialized_variables/VarIsInitializedOp_179"
  input: "report_uninitialized_variables/VarIsInitializedOp_180"
  input: "report_uninitialized_variables/VarIsInitializedOp_181"
  input: "report_uninitialized_variables/VarIsInitializedOp_182"
  input: "report_uninitialized_variables/VarIsInitializedOp_183"
  input: "report_uninitialized_variables/VarIsInitializedOp_184"
  input: "report_uninitialized_variables/VarIsInitializedOp_185"
  input: "report_uninitialized_variables/VarIsInitializedOp_186"
  input: "report_uninitialized_variables/VarIsInitializedOp_187"
  input: "report_uninitialized_variables/VarIsInitializedOp_188"
  input: "report_uninitialized_variables/VarIsInitializedOp_189"
  input: "report_uninitialized_variables/VarIsInitializedOp_190"
  input: "report_uninitialized_variables/VarIsInitializedOp_191"
  input: "report_uninitialized_variables/VarIsInitializedOp_192"
  input: "report_uninitialized_variables/VarIsInitializedOp_193"
  input: "report_uninitialized_variables/VarIsInitializedOp_194"
  input: "report_uninitialized_variables/VarIsInitializedOp_195"
  input: "report_uninitialized_variables/VarIsInitializedOp_196"
  input: "report_uninitialized_variables/VarIsInitializedOp_197"
  input: "report_uninitialized_variables/VarIsInitializedOp_198"
  input: "report_uninitialized_variables/VarIsInitializedOp_199"
  input: "report_uninitialized_variables/VarIsInitializedOp_200"
  input: "report_uninitialized_variables/VarIsInitializedOp_201"
  input: "report_uninitialized_variables/VarIsInitializedOp_202"
  input: "report_uninitialized_variables/VarIsInitializedOp_203"
  input: "report_uninitialized_variables/VarIsInitializedOp_204"
  input: "report_uninitialized_variables/VarIsInitializedOp_205"
  input: "report_uninitialized_variables/VarIsInitializedOp_206"
  input: "report_uninitialized_variables/VarIsInitializedOp_207"
  input: "report_uninitialized_variables/VarIsInitializedOp_208"
  input: "report_uninitialized_variables/VarIsInitializedOp_209"
  input: "report_uninitialized_variables/VarIsInitializedOp_210"
  input: "report_uninitialized_variables/VarIsInitializedOp_211"
  input: "report_uninitialized_variables/VarIsInitializedOp_212"
  input: "report_uninitialized_variables/VarIsInitializedOp_213"
  input: "report_uninitialized_variables/VarIsInitializedOp_214"
  input: "report_uninitialized_variables/VarIsInitializedOp_215"
  input: "report_uninitialized_variables/VarIsInitializedOp_216"
  input: "report_uninitialized_variables/VarIsInitializedOp_217"
  input: "report_uninitialized_variables/VarIsInitializedOp_218"
  input: "report_uninitialized_variables/VarIsInitializedOp_219"
  input: "report_uninitialized_variables/VarIsInitializedOp_220"
  input: "report_uninitialized_variables/VarIsInitializedOp_221"
  input: "report_uninitialized_variables/VarIsInitializedOp_222"
  input: "report_uninitialized_variables/VarIsInitializedOp_223"
  input: "report_uninitialized_variables/VarIsInitializedOp_224"
  input: "report_uninitialized_variables/VarIsInitializedOp_225"
  input: "report_uninitialized_variables/VarIsInitializedOp_226"
  input: "report_uninitialized_variables/VarIsInitializedOp_227"
  input: "report_uninitialized_variables/VarIsInitializedOp_228"
  input: "report_uninitialized_variables/VarIsInitializedOp_229"
  input: "report_uninitialized_variables/VarIsInitializedOp_230"
  input: "report_uninitialized_variables/VarIsInitializedOp_231"
  input: "report_uninitialized_variables/VarIsInitializedOp_232"
  input: "report_uninitialized_variables/VarIsInitializedOp_233"
  input: "report_uninitialized_variables/VarIsInitializedOp_234"
  input: "report_uninitialized_variables/VarIsInitializedOp_235"
  input: "report_uninitialized_variables/VarIsInitializedOp_236"
  input: "report_uninitialized_variables/VarIsInitializedOp_237"
  input: "report_uninitialized_variables/VarIsInitializedOp_238"
  input: "report_uninitialized_variables/VarIsInitializedOp_239"
  input: "report_uninitialized_variables/VarIsInitializedOp_240"
  input: "report_uninitialized_variables/VarIsInitializedOp_241"
  input: "report_uninitialized_variables/VarIsInitializedOp_242"
  input: "report_uninitialized_variables/VarIsInitializedOp_243"
  input: "report_uninitialized_variables/VarIsInitializedOp_244"
  input: "report_uninitialized_variables/VarIsInitializedOp_245"
  input: "report_uninitialized_variables/VarIsInitializedOp_246"
  input: "report_uninitialized_variables/VarIsInitializedOp_247"
  input: "report_uninitialized_variables/VarIsInitializedOp_248"
  input: "report_uninitialized_variables/VarIsInitializedOp_249"
  input: "report_uninitialized_variables/VarIsInitializedOp_250"
  input: "report_uninitialized_variables/VarIsInitializedOp_251"
  input: "report_uninitialized_variables/VarIsInitializedOp_252"
  input: "report_uninitialized_variables/VarIsInitializedOp_253"
  input: "report_uninitialized_variables/VarIsInitializedOp_254"
  input: "report_uninitialized_variables/VarIsInitializedOp_255"
  input: "report_uninitialized_variables/VarIsInitializedOp_256"
  input: "report_uninitialized_variables/VarIsInitializedOp_257"
  input: "report_uninitialized_variables/VarIsInitializedOp_258"
  input: "report_uninitialized_variables/VarIsInitializedOp_259"
  input: "report_uninitialized_variables/VarIsInitializedOp_260"
  input: "report_uninitialized_variables/VarIsInitializedOp_261"
  input: "report_uninitialized_variables/VarIsInitializedOp_262"
  input: "report_uninitialized_variables/VarIsInitializedOp_263"
  input: "report_uninitialized_variables/VarIsInitializedOp_264"
  input: "report_uninitialized_variables/VarIsInitializedOp_265"
  input: "report_uninitialized_variables/VarIsInitializedOp_266"
  input: "report_uninitialized_variables/VarIsInitializedOp_267"
  input: "report_uninitialized_variables/VarIsInitializedOp_268"
  input: "report_uninitialized_variables/VarIsInitializedOp_269"
  input: "report_uninitialized_variables/VarIsInitializedOp_270"
  input: "report_uninitialized_variables/VarIsInitializedOp_271"
  input: "report_uninitialized_variables/VarIsInitializedOp_272"
  input: "report_uninitialized_variables/VarIsInitializedOp_273"
  input: "report_uninitialized_variables/VarIsInitializedOp_274"
  input: "report_uninitialized_variables/VarIsInitializedOp_275"
  input: "report_uninitialized_variables/VarIsInitializedOp_276"
  input: "report_uninitialized_variables/VarIsInitializedOp_277"
  input: "report_uninitialized_variables/VarIsInitializedOp_278"
  input: "report_uninitialized_variables/VarIsInitializedOp_279"
  input: "report_uninitialized_variables/VarIsInitializedOp_280"
  input: "report_uninitialized_variables/VarIsInitializedOp_281"
  input: "report_uninitialized_variables/VarIsInitializedOp_282"
  input: "report_uninitialized_variables/VarIsInitializedOp_283"
  input: "report_uninitialized_variables/VarIsInitializedOp_284"
  input: "report_uninitialized_variables/VarIsInitializedOp_285"
  input: "report_uninitialized_variables/VarIsInitializedOp_286"
  input: "report_uninitialized_variables/VarIsInitializedOp_287"
  input: "report_uninitialized_variables/VarIsInitializedOp_288"
  input: "report_uninitialized_variables/VarIsInitializedOp_289"
  input: "report_uninitialized_variables/VarIsInitializedOp_290"
  input: "report_uninitialized_variables/VarIsInitializedOp_291"
  input: "report_uninitialized_variables/VarIsInitializedOp_292"
  input: "report_uninitialized_variables/VarIsInitializedOp_293"
  input: "report_uninitialized_variables/VarIsInitializedOp_294"
  input: "report_uninitialized_variables/VarIsInitializedOp_295"
  input: "report_uninitialized_variables/VarIsInitializedOp_296"
  input: "report_uninitialized_variables/VarIsInitializedOp_297"
  input: "report_uninitialized_variables/VarIsInitializedOp_298"
  input: "report_uninitialized_variables/VarIsInitializedOp_299"
  input: "report_uninitialized_variables/VarIsInitializedOp_300"
  input: "report_uninitialized_variables/VarIsInitializedOp_301"
  input: "report_uninitialized_variables/VarIsInitializedOp_302"
  input: "report_uninitialized_variables/VarIsInitializedOp_303"
  input: "report_uninitialized_variables/VarIsInitializedOp_304"
  input: "report_uninitialized_variables/VarIsInitializedOp_305"
  input: "report_uninitialized_variables/VarIsInitializedOp_306"
  input: "report_uninitialized_variables/VarIsInitializedOp_307"
  input: "report_uninitialized_variables/VarIsInitializedOp_308"
  input: "report_uninitialized_variables/VarIsInitializedOp_309"
  input: "report_uninitialized_variables/VarIsInitializedOp_310"
  input: "report_uninitialized_variables/VarIsInitializedOp_311"
  input: "report_uninitialized_variables/VarIsInitializedOp_312"
  input: "report_uninitialized_variables/VarIsInitializedOp_313"
  input: "report_uninitialized_variables/VarIsInitializedOp_314"
  input: "report_uninitialized_variables/VarIsInitializedOp_315"
  input: "report_uninitialized_variables/VarIsInitializedOp_316"
  input: "report_uninitialized_variables/VarIsInitializedOp_317"
  input: "report_uninitialized_variables/VarIsInitializedOp_318"
  input: "report_uninitialized_variables/VarIsInitializedOp_319"
  input: "report_uninitialized_variables/VarIsInitializedOp_320"
  input: "report_uninitialized_variables/VarIsInitializedOp_321"
  input: "report_uninitialized_variables/VarIsInitializedOp_322"
  input: "report_uninitialized_variables/VarIsInitializedOp_323"
  input: "report_uninitialized_variables/VarIsInitializedOp_324"
  input: "report_uninitialized_variables/VarIsInitializedOp_325"
  input: "report_uninitialized_variables/VarIsInitializedOp_326"
  input: "report_uninitialized_variables/VarIsInitializedOp_327"
  input: "report_uninitialized_variables/VarIsInitializedOp_328"
  input: "report_uninitialized_variables/VarIsInitializedOp_329"
  input: "report_uninitialized_variables/VarIsInitializedOp_330"
  input: "report_uninitialized_variables/VarIsInitializedOp_331"
  input: "report_uninitialized_variables/VarIsInitializedOp_332"
  input: "report_uninitialized_variables/VarIsInitializedOp_333"
  input: "report_uninitialized_variables/VarIsInitializedOp_334"
  input: "report_uninitialized_variables/VarIsInitializedOp_335"
  input: "report_uninitialized_variables/VarIsInitializedOp_336"
  input: "report_uninitialized_variables/VarIsInitializedOp_337"
  input: "report_uninitialized_variables/VarIsInitializedOp_338"
  input: "report_uninitialized_variables/VarIsInitializedOp_339"
  input: "report_uninitialized_variables/VarIsInitializedOp_340"
  input: "report_uninitialized_variables/VarIsInitializedOp_341"
  input: "report_uninitialized_variables/VarIsInitializedOp_342"
  input: "report_uninitialized_variables/VarIsInitializedOp_343"
  input: "report_uninitialized_variables/VarIsInitializedOp_344"
  input: "report_uninitialized_variables/VarIsInitializedOp_345"
  input: "report_uninitialized_variables/VarIsInitializedOp_346"
  input: "report_uninitialized_variables/VarIsInitializedOp_347"
  input: "report_uninitialized_variables/VarIsInitializedOp_348"
  input: "report_uninitialized_variables/VarIsInitializedOp_349"
  input: "report_uninitialized_variables/VarIsInitializedOp_350"
  input: "report_uninitialized_variables/VarIsInitializedOp_351"
  input: "report_uninitialized_variables/VarIsInitializedOp_352"
  input: "report_uninitialized_variables/VarIsInitializedOp_353"
  input: "report_uninitialized_variables/VarIsInitializedOp_354"
  input: "report_uninitialized_variables/VarIsInitializedOp_355"
  input: "report_uninitialized_variables/VarIsInitializedOp_356"
  input: "report_uninitialized_variables/VarIsInitializedOp_357"
  input: "report_uninitialized_variables/VarIsInitializedOp_358"
  input: "report_uninitialized_variables/VarIsInitializedOp_359"
  input: "report_uninitialized_variables/VarIsInitializedOp_360"
  input: "report_uninitialized_variables/VarIsInitializedOp_361"
  input: "report_uninitialized_variables/VarIsInitializedOp_362"
  input: "report_uninitialized_variables/VarIsInitializedOp_363"
  input: "report_uninitialized_variables/VarIsInitializedOp_364"
  input: "report_uninitialized_variables/VarIsInitializedOp_365"
  input: "report_uninitialized_variables/VarIsInitializedOp_366"
  input: "report_uninitialized_variables/VarIsInitializedOp_367"
  input: "report_uninitialized_variables/VarIsInitializedOp_368"
  input: "report_uninitialized_variables/VarIsInitializedOp_369"
  input: "report_uninitialized_variables/VarIsInitializedOp_370"
  input: "report_uninitialized_variables/VarIsInitializedOp_371"
  input: "report_uninitialized_variables/VarIsInitializedOp_372"
  input: "report_uninitialized_variables/VarIsInitializedOp_373"
  input: "report_uninitialized_variables/VarIsInitializedOp_374"
  input: "report_uninitialized_variables/VarIsInitializedOp_375"
  input: "report_uninitialized_variables/VarIsInitializedOp_376"
  input: "report_uninitialized_variables/VarIsInitializedOp_377"
  input: "report_uninitialized_variables/VarIsInitializedOp_378"
  input: "report_uninitialized_variables/VarIsInitializedOp_379"
  input: "report_uninitialized_variables/VarIsInitializedOp_380"
  input: "report_uninitialized_variables/VarIsInitializedOp_381"
  input: "report_uninitialized_variables/VarIsInitializedOp_382"
  input: "report_uninitialized_variables/VarIsInitializedOp_383"
  input: "report_uninitialized_variables/VarIsInitializedOp_384"
  input: "report_uninitialized_variables/VarIsInitializedOp_385"
  input: "report_uninitialized_variables/VarIsInitializedOp_386"
  input: "report_uninitialized_variables/VarIsInitializedOp_387"
  input: "report_uninitialized_variables/VarIsInitializedOp_388"
  input: "report_uninitialized_variables/VarIsInitializedOp_389"
  input: "report_uninitialized_variables/VarIsInitializedOp_390"
  input: "report_uninitialized_variables/VarIsInitializedOp_391"
  input: "report_uninitialized_variables/VarIsInitializedOp_392"
  input: "report_uninitialized_variables/VarIsInitializedOp_393"
  input: "report_uninitialized_variables/VarIsInitializedOp_394"
  input: "report_uninitialized_variables/VarIsInitializedOp_395"
  input: "report_uninitialized_variables/VarIsInitializedOp_396"
  input: "report_uninitialized_variables/VarIsInitializedOp_397"
  input: "report_uninitialized_variables/VarIsInitializedOp_398"
  input: "report_uninitialized_variables/VarIsInitializedOp_399"
  input: "report_uninitialized_variables/VarIsInitializedOp_400"
  input: "report_uninitialized_variables/VarIsInitializedOp_401"
  input: "report_uninitialized_variables/VarIsInitializedOp_402"
  input: "report_uninitialized_variables/VarIsInitializedOp_403"
  input: "report_uninitialized_variables/VarIsInitializedOp_404"
  input: "report_uninitialized_variables/VarIsInitializedOp_405"
  input: "report_uninitialized_variables/VarIsInitializedOp_406"
  input: "report_uninitialized_variables/VarIsInitializedOp_407"
  input: "report_uninitialized_variables/VarIsInitializedOp_408"
  input: "report_uninitialized_variables/VarIsInitializedOp_409"
  input: "report_uninitialized_variables/VarIsInitializedOp_410"
  input: "report_uninitialized_variables/VarIsInitializedOp_411"
  input: "report_uninitialized_variables/VarIsInitializedOp_412"
  input: "report_uninitialized_variables/VarIsInitializedOp_413"
  input: "report_uninitialized_variables/VarIsInitializedOp_414"
  input: "report_uninitialized_variables/VarIsInitializedOp_415"
  input: "report_uninitialized_variables/VarIsInitializedOp_416"
  input: "report_uninitialized_variables/VarIsInitializedOp_417"
  input: "report_uninitialized_variables/VarIsInitializedOp_418"
  input: "report_uninitialized_variables/VarIsInitializedOp_419"
  input: "report_uninitialized_variables/VarIsInitializedOp_420"
  input: "report_uninitialized_variables/VarIsInitializedOp_421"
  input: "report_uninitialized_variables/VarIsInitializedOp_422"
  input: "report_uninitialized_variables/VarIsInitializedOp_423"
  input: "report_uninitialized_variables/VarIsInitializedOp_424"
  input: "report_uninitialized_variables/VarIsInitializedOp_425"
  input: "report_uninitialized_variables/VarIsInitializedOp_426"
  input: "report_uninitialized_variables/VarIsInitializedOp_427"
  input: "report_uninitialized_variables/VarIsInitializedOp_428"
  input: "report_uninitialized_variables/VarIsInitializedOp_429"
  input: "report_uninitialized_variables/VarIsInitializedOp_430"
  input: "report_uninitialized_variables/VarIsInitializedOp_431"
  input: "report_uninitialized_variables/VarIsInitializedOp_432"
  input: "report_uninitialized_variables/VarIsInitializedOp_433"
  input: "report_uninitialized_variables/VarIsInitializedOp_434"
  input: "report_uninitialized_variables/VarIsInitializedOp_435"
  input: "report_uninitialized_variables/VarIsInitializedOp_436"
  input: "report_uninitialized_variables/VarIsInitializedOp_437"
  input: "report_uninitialized_variables/VarIsInitializedOp_438"
  input: "report_uninitialized_variables/VarIsInitializedOp_439"
  input: "report_uninitialized_variables/VarIsInitializedOp_440"
  input: "report_uninitialized_variables/VarIsInitializedOp_441"
  input: "report_uninitialized_variables/VarIsInitializedOp_442"
  input: "report_uninitialized_variables/VarIsInitializedOp_443"
  input: "report_uninitialized_variables/VarIsInitializedOp_444"
  input: "report_uninitialized_variables/VarIsInitializedOp_445"
  input: "report_uninitialized_variables/VarIsInitializedOp_446"
  input: "report_uninitialized_variables/VarIsInitializedOp_447"
  input: "report_uninitialized_variables/VarIsInitializedOp_448"
  input: "report_uninitialized_variables/VarIsInitializedOp_449"
  input: "report_uninitialized_variables/VarIsInitializedOp_450"
  input: "report_uninitialized_variables/VarIsInitializedOp_451"
  input: "report_uninitialized_variables/VarIsInitializedOp_452"
  input: "report_uninitialized_variables/VarIsInitializedOp_453"
  input: "report_uninitialized_variables/VarIsInitializedOp_454"
  input: "report_uninitialized_variables/VarIsInitializedOp_455"
  input: "report_uninitialized_variables/VarIsInitializedOp_456"
  input: "report_uninitialized_variables/VarIsInitializedOp_457"
  input: "report_uninitialized_variables/VarIsInitializedOp_458"
  input: "report_uninitialized_variables/VarIsInitializedOp_459"
  input: "report_uninitialized_variables/VarIsInitializedOp_460"
  input: "report_uninitialized_variables/VarIsInitializedOp_461"
  input: "report_uninitialized_variables/VarIsInitializedOp_462"
  input: "report_uninitialized_variables/VarIsInitializedOp_463"
  input: "report_uninitialized_variables/VarIsInitializedOp_464"
  input: "report_uninitialized_variables/VarIsInitializedOp_465"
  input: "report_uninitialized_variables/VarIsInitializedOp_466"
  input: "report_uninitialized_variables/VarIsInitializedOp_467"
  input: "report_uninitialized_variables/VarIsInitializedOp_468"
  input: "report_uninitialized_variables/VarIsInitializedOp_469"
  input: "report_uninitialized_variables/VarIsInitializedOp_470"
  input: "report_uninitialized_variables/VarIsInitializedOp_471"
  input: "report_uninitialized_variables/VarIsInitializedOp_472"
  input: "report_uninitialized_variables/VarIsInitializedOp_473"
  input: "report_uninitialized_variables/VarIsInitializedOp_474"
  input: "report_uninitialized_variables/VarIsInitializedOp_475"
  input: "report_uninitialized_variables/VarIsInitializedOp_476"
  input: "report_uninitialized_variables/VarIsInitializedOp_477"
  input: "report_uninitialized_variables/VarIsInitializedOp_478"
  input: "report_uninitialized_variables/VarIsInitializedOp_479"
  input: "report_uninitialized_variables/VarIsInitializedOp_480"
  input: "report_uninitialized_variables/VarIsInitializedOp_481"
  input: "report_uninitialized_variables/VarIsInitializedOp_482"
  input: "report_uninitialized_variables/VarIsInitializedOp_483"
  input: "report_uninitialized_variables/VarIsInitializedOp_484"
  input: "report_uninitialized_variables/VarIsInitializedOp_485"
  input: "report_uninitialized_variables/VarIsInitializedOp_486"
  input: "report_uninitialized_variables/VarIsInitializedOp_487"
  input: "report_uninitialized_variables/VarIsInitializedOp_488"
  input: "report_uninitialized_variables/VarIsInitializedOp_489"
  input: "report_uninitialized_variables/VarIsInitializedOp_490"
  input: "report_uninitialized_variables/VarIsInitializedOp_491"
  input: "report_uninitialized_variables/VarIsInitializedOp_492"
  input: "report_uninitialized_variables/VarIsInitializedOp_493"
  input: "report_uninitialized_variables/VarIsInitializedOp_494"
  input: "report_uninitialized_variables/VarIsInitializedOp_495"
  input: "report_uninitialized_variables/VarIsInitializedOp_496"
  input: "report_uninitialized_variables/VarIsInitializedOp_497"
  input: "report_uninitialized_variables/VarIsInitializedOp_498"
  input: "report_uninitialized_variables/VarIsInitializedOp_499"
  input: "report_uninitialized_variables/VarIsInitializedOp_500"
  input: "report_uninitialized_variables/VarIsInitializedOp_501"
  input: "report_uninitialized_variables/VarIsInitializedOp_502"
  input: "report_uninitialized_variables/VarIsInitializedOp_503"
  input: "report_uninitialized_variables/VarIsInitializedOp_504"
  input: "report_uninitialized_variables/VarIsInitializedOp_505"
  input: "report_uninitialized_variables/VarIsInitializedOp_506"
  input: "report_uninitialized_variables/VarIsInitializedOp_507"
  input: "report_uninitialized_variables/VarIsInitializedOp_508"
  input: "report_uninitialized_variables/VarIsInitializedOp_509"
  input: "report_uninitialized_variables/VarIsInitializedOp_510"
  input: "report_uninitialized_variables/VarIsInitializedOp_511"
  input: "report_uninitialized_variables/VarIsInitializedOp_512"
  input: "report_uninitialized_variables/VarIsInitializedOp_513"
  input: "report_uninitialized_variables/VarIsInitializedOp_514"
  input: "report_uninitialized_variables/VarIsInitializedOp_515"
  input: "report_uninitialized_variables/VarIsInitializedOp_516"
  input: "report_uninitialized_variables/VarIsInitializedOp_517"
  input: "report_uninitialized_variables/VarIsInitializedOp_518"
  input: "report_uninitialized_variables/VarIsInitializedOp_519"
  input: "report_uninitialized_variables/VarIsInitializedOp_520"
  input: "report_uninitialized_variables/VarIsInitializedOp_521"
  input: "report_uninitialized_variables/VarIsInitializedOp_522"
  input: "report_uninitialized_variables/VarIsInitializedOp_523"
  input: "report_uninitialized_variables/VarIsInitializedOp_524"
  input: "report_uninitialized_variables/VarIsInitializedOp_525"
  input: "report_uninitialized_variables/VarIsInitializedOp_526"
  input: "report_uninitialized_variables/VarIsInitializedOp_527"
  input: "report_uninitialized_variables/VarIsInitializedOp_528"
  input: "report_uninitialized_variables/VarIsInitializedOp_529"
  input: "report_uninitialized_variables/VarIsInitializedOp_530"
  input: "report_uninitialized_variables/VarIsInitializedOp_531"
  input: "report_uninitialized_variables/VarIsInitializedOp_532"
  input: "report_uninitialized_variables/VarIsInitializedOp_533"
  input: "report_uninitialized_variables/VarIsInitializedOp_534"
  input: "report_uninitialized_variables/VarIsInitializedOp_535"
  input: "report_uninitialized_variables/VarIsInitializedOp_536"
  input: "report_uninitialized_variables/VarIsInitializedOp_537"
  input: "report_uninitialized_variables/VarIsInitializedOp_538"
  input: "report_uninitialized_variables/VarIsInitializedOp_539"
  input: "report_uninitialized_variables/VarIsInitializedOp_540"
  input: "report_uninitialized_variables/VarIsInitializedOp_541"
  input: "report_uninitialized_variables/VarIsInitializedOp_542"
  input: "report_uninitialized_variables/VarIsInitializedOp_543"
  input: "report_uninitialized_variables/VarIsInitializedOp_544"
  input: "report_uninitialized_variables/VarIsInitializedOp_545"
  input: "report_uninitialized_variables/VarIsInitializedOp_546"
  input: "report_uninitialized_variables/VarIsInitializedOp_547"
  input: "report_uninitialized_variables/VarIsInitializedOp_548"
  input: "report_uninitialized_variables/VarIsInitializedOp_549"
  input: "report_uninitialized_variables/VarIsInitializedOp_550"
  input: "report_uninitialized_variables/VarIsInitializedOp_551"
  input: "report_uninitialized_variables/VarIsInitializedOp_552"
  input: "report_uninitialized_variables/VarIsInitializedOp_553"
  input: "report_uninitialized_variables/VarIsInitializedOp_554"
  input: "report_uninitialized_variables/VarIsInitializedOp_555"
  input: "report_uninitialized_variables/VarIsInitializedOp_556"
  input: "report_uninitialized_variables/VarIsInitializedOp_557"
  input: "report_uninitialized_variables/VarIsInitializedOp_558"
  input: "report_uninitialized_variables/VarIsInitializedOp_559"
  input: "report_uninitialized_variables/VarIsInitializedOp_560"
  input: "report_uninitialized_variables/VarIsInitializedOp_561"
  input: "report_uninitialized_variables/VarIsInitializedOp_562"
  input: "report_uninitialized_variables/VarIsInitializedOp_563"
  input: "report_uninitialized_variables/VarIsInitializedOp_564"
  input: "report_uninitialized_variables/VarIsInitializedOp_565"
  input: "report_uninitialized_variables/VarIsInitializedOp_566"
  input: "report_uninitialized_variables/VarIsInitializedOp_567"
  input: "report_uninitialized_variables/VarIsInitializedOp_568"
  input: "report_uninitialized_variables/VarIsInitializedOp_569"
  input: "report_uninitialized_variables/VarIsInitializedOp_570"
  input: "report_uninitialized_variables/VarIsInitializedOp_571"
  input: "report_uninitialized_variables/VarIsInitializedOp_572"
  input: "report_uninitialized_variables/VarIsInitializedOp_573"
  input: "report_uninitialized_variables/VarIsInitializedOp_574"
  input: "report_uninitialized_variables/VarIsInitializedOp_575"
  input: "report_uninitialized_variables/VarIsInitializedOp_576"
  input: "report_uninitialized_variables/VarIsInitializedOp_577"
  input: "report_uninitialized_variables/VarIsInitializedOp_578"
  input: "report_uninitialized_variables/VarIsInitializedOp_579"
  input: "report_uninitialized_variables/VarIsInitializedOp_580"
  input: "report_uninitialized_variables/VarIsInitializedOp_581"
  input: "report_uninitialized_variables/VarIsInitializedOp_582"
  input: "report_uninitialized_variables/VarIsInitializedOp_583"
  input: "report_uninitialized_variables/VarIsInitializedOp_584"
  input: "report_uninitialized_variables/VarIsInitializedOp_585"
  input: "report_uninitialized_variables/VarIsInitializedOp_586"
  input: "report_uninitialized_variables/VarIsInitializedOp_587"
  input: "report_uninitialized_variables/VarIsInitializedOp_588"
  input: "report_uninitialized_variables/VarIsInitializedOp_589"
  input: "report_uninitialized_variables/VarIsInitializedOp_590"
  input: "report_uninitialized_variables/VarIsInitializedOp_591"
  input: "report_uninitialized_variables/VarIsInitializedOp_592"
  input: "report_uninitialized_variables/VarIsInitializedOp_593"
  input: "report_uninitialized_variables/VarIsInitializedOp_594"
  input: "report_uninitialized_variables/VarIsInitializedOp_595"
  input: "report_uninitialized_variables/VarIsInitializedOp_596"
  input: "report_uninitialized_variables/VarIsInitializedOp_597"
  input: "report_uninitialized_variables/VarIsInitializedOp_598"
  input: "report_uninitialized_variables/VarIsInitializedOp_599"
  input: "report_uninitialized_variables/VarIsInitializedOp_600"
  input: "report_uninitialized_variables/VarIsInitializedOp_601"
  input: "report_uninitialized_variables/VarIsInitializedOp_602"
  input: "report_uninitialized_variables/VarIsInitializedOp_603"
  input: "report_uninitialized_variables/VarIsInitializedOp_604"
  input: "report_uninitialized_variables/VarIsInitializedOp_605"
  input: "report_uninitialized_variables/VarIsInitializedOp_606"
  input: "report_uninitialized_variables/VarIsInitializedOp_607"
  input: "report_uninitialized_variables/VarIsInitializedOp_608"
  input: "report_uninitialized_variables/VarIsInitializedOp_609"
  input: "report_uninitialized_variables/VarIsInitializedOp_610"
  input: "report_uninitialized_variables/VarIsInitializedOp_611"
  input: "report_uninitialized_variables/VarIsInitializedOp_612"
  input: "report_uninitialized_variables/VarIsInitializedOp_613"
  input: "report_uninitialized_variables/VarIsInitializedOp_614"
  input: "report_uninitialized_variables/VarIsInitializedOp_615"
  input: "report_uninitialized_variables/VarIsInitializedOp_616"
  input: "report_uninitialized_variables/VarIsInitializedOp_617"
  input: "report_uninitialized_variables/VarIsInitializedOp_618"
  input: "report_uninitialized_variables/VarIsInitializedOp_619"
  input: "report_uninitialized_variables/VarIsInitializedOp_620"
  input: "report_uninitialized_variables/VarIsInitializedOp_621"
  input: "report_uninitialized_variables/VarIsInitializedOp_622"
  input: "report_uninitialized_variables/VarIsInitializedOp_623"
  input: "report_uninitialized_variables/VarIsInitializedOp_624"
  input: "report_uninitialized_variables/VarIsInitializedOp_625"
  input: "report_uninitialized_variables/VarIsInitializedOp_626"
  input: "report_uninitialized_variables/VarIsInitializedOp_627"
  input: "report_uninitialized_variables/VarIsInitializedOp_628"
  input: "report_uninitialized_variables/VarIsInitializedOp_629"
  input: "report_uninitialized_variables/VarIsInitializedOp_630"
  input: "report_uninitialized_variables/VarIsInitializedOp_631"
  input: "report_uninitialized_variables/VarIsInitializedOp_632"
  input: "report_uninitialized_variables/VarIsInitializedOp_633"
  input: "report_uninitialized_variables/VarIsInitializedOp_634"
  input: "report_uninitialized_variables/VarIsInitializedOp_635"
  input: "report_uninitialized_variables/VarIsInitializedOp_636"
  input: "report_uninitialized_variables/VarIsInitializedOp_637"
  input: "report_uninitialized_variables/VarIsInitializedOp_638"
  input: "report_uninitialized_variables/VarIsInitializedOp_639"
  input: "report_uninitialized_variables/VarIsInitializedOp_640"
  input: "report_uninitialized_variables/VarIsInitializedOp_641"
  input: "report_uninitialized_variables/VarIsInitializedOp_642"
  input: "report_uninitialized_variables/VarIsInitializedOp_643"
  input: "report_uninitialized_variables/VarIsInitializedOp_644"
  input: "report_uninitialized_variables/VarIsInitializedOp_645"
  input: "report_uninitialized_variables/VarIsInitializedOp_646"
  input: "report_uninitialized_variables/VarIsInitializedOp_647"
  input: "report_uninitialized_variables/VarIsInitializedOp_648"
  input: "report_uninitialized_variables/VarIsInitializedOp_649"
  input: "report_uninitialized_variables/VarIsInitializedOp_650"
  input: "report_uninitialized_variables/VarIsInitializedOp_651"
  input: "report_uninitialized_variables/VarIsInitializedOp_652"
  input: "report_uninitialized_variables/VarIsInitializedOp_653"
  input: "report_uninitialized_variables/VarIsInitializedOp_654"
  input: "report_uninitialized_variables/VarIsInitializedOp_655"
  input: "report_uninitialized_variables/VarIsInitializedOp_656"
  input: "report_uninitialized_variables/VarIsInitializedOp_657"
  input: "report_uninitialized_variables/VarIsInitializedOp_658"
  input: "report_uninitialized_variables/VarIsInitializedOp_659"
  input: "report_uninitialized_variables/VarIsInitializedOp_660"
  input: "report_uninitialized_variables/VarIsInitializedOp_661"
  input: "report_uninitialized_variables/VarIsInitializedOp_662"
  input: "report_uninitialized_variables/VarIsInitializedOp_663"
  input: "report_uninitialized_variables/VarIsInitializedOp_664"
  input: "report_uninitialized_variables/VarIsInitializedOp_665"
  input: "report_uninitialized_variables/VarIsInitializedOp_666"
  input: "report_uninitialized_variables/VarIsInitializedOp_667"
  input: "report_uninitialized_variables/VarIsInitializedOp_668"
  input: "report_uninitialized_variables/VarIsInitializedOp_669"
  input: "report_uninitialized_variables/VarIsInitializedOp_670"
  input: "report_uninitialized_variables/VarIsInitializedOp_671"
  input: "report_uninitialized_variables/VarIsInitializedOp_672"
  input: "report_uninitialized_variables/VarIsInitializedOp_673"
  input: "report_uninitialized_variables/VarIsInitializedOp_674"
  input: "report_uninitialized_variables/VarIsInitializedOp_675"
  input: "report_uninitialized_variables/VarIsInitializedOp_676"
  input: "report_uninitialized_variables/VarIsInitializedOp_677"
  input: "report_uninitialized_variables/VarIsInitializedOp_678"
  input: "report_uninitialized_variables/VarIsInitializedOp_679"
  input: "report_uninitialized_variables/VarIsInitializedOp_680"
  input: "report_uninitialized_variables/VarIsInitializedOp_681"
  input: "report_uninitialized_variables/VarIsInitializedOp_682"
  input: "report_uninitialized_variables/VarIsInitializedOp_683"
  input: "report_uninitialized_variables/VarIsInitializedOp_684"
  input: "report_uninitialized_variables/VarIsInitializedOp_685"
  input: "report_uninitialized_variables/VarIsInitializedOp_686"
  input: "report_uninitialized_variables/VarIsInitializedOp_687"
  input: "report_uninitialized_variables/VarIsInitializedOp_688"
  input: "report_uninitialized_variables/VarIsInitializedOp_689"
  input: "report_uninitialized_variables/VarIsInitializedOp_690"
  input: "report_uninitialized_variables/VarIsInitializedOp_691"
  input: "report_uninitialized_variables/VarIsInitializedOp_692"
  input: "report_uninitialized_variables/VarIsInitializedOp_693"
  input: "report_uninitialized_variables/VarIsInitializedOp_694"
  input: "report_uninitialized_variables/VarIsInitializedOp_695"
  input: "report_uninitialized_variables/VarIsInitializedOp_696"
  input: "report_uninitialized_variables/VarIsInitializedOp_697"
  input: "report_uninitialized_variables/VarIsInitializedOp_698"
  input: "report_uninitialized_variables/VarIsInitializedOp_699"
  input: "report_uninitialized_variables/VarIsInitializedOp_700"
  input: "report_uninitialized_variables/VarIsInitializedOp_701"
  input: "report_uninitialized_variables/VarIsInitializedOp_702"
  input: "report_uninitialized_variables/VarIsInitializedOp_703"
  input: "report_uninitialized_variables/VarIsInitializedOp_704"
  input: "report_uninitialized_variables/VarIsInitializedOp_705"
  input: "report_uninitialized_variables/VarIsInitializedOp_706"
  input: "report_uninitialized_variables/VarIsInitializedOp_707"
  input: "report_uninitialized_variables/VarIsInitializedOp_708"
  input: "report_uninitialized_variables/VarIsInitializedOp_709"
  input: "report_uninitialized_variables/VarIsInitializedOp_710"
  input: "report_uninitialized_variables/VarIsInitializedOp_711"
  input: "report_uninitialized_variables/VarIsInitializedOp_712"
  input: "report_uninitialized_variables/VarIsInitializedOp_713"
  input: "report_uninitialized_variables/VarIsInitializedOp_714"
  input: "report_uninitialized_variables/VarIsInitializedOp_715"
  input: "report_uninitialized_variables/VarIsInitializedOp_716"
  input: "report_uninitialized_variables/VarIsInitializedOp_717"
  input: "report_uninitialized_variables/VarIsInitializedOp_718"
  input: "report_uninitialized_variables/VarIsInitializedOp_719"
  input: "report_uninitialized_variables/VarIsInitializedOp_720"
  input: "report_uninitialized_variables/VarIsInitializedOp_721"
  input: "report_uninitialized_variables/VarIsInitializedOp_722"
  input: "report_uninitialized_variables/VarIsInitializedOp_723"
  input: "report_uninitialized_variables/VarIsInitializedOp_724"
  input: "report_uninitialized_variables/VarIsInitializedOp_725"
  input: "report_uninitialized_variables/VarIsInitializedOp_726"
  input: "report_uninitialized_variables/VarIsInitializedOp_727"
  input: "report_uninitialized_variables/VarIsInitializedOp_728"
  input: "report_uninitialized_variables/VarIsInitializedOp_729"
  input: "report_uninitialized_variables/VarIsInitializedOp_730"
  input: "report_uninitialized_variables/VarIsInitializedOp_731"
  input: "report_uninitialized_variables/VarIsInitializedOp_732"
  input: "report_uninitialized_variables/VarIsInitializedOp_733"
  input: "report_uninitialized_variables/VarIsInitializedOp_734"
  input: "report_uninitialized_variables/VarIsInitializedOp_735"
  input: "report_uninitialized_variables/VarIsInitializedOp_736"
  input: "report_uninitialized_variables/VarIsInitializedOp_737"
  input: "report_uninitialized_variables/VarIsInitializedOp_738"
  input: "report_uninitialized_variables/VarIsInitializedOp_739"
  input: "report_uninitialized_variables/VarIsInitializedOp_740"
  input: "report_uninitialized_variables/VarIsInitializedOp_741"
  input: "report_uninitialized_variables/VarIsInitializedOp_742"
  input: "report_uninitialized_variables/VarIsInitializedOp_743"
  input: "report_uninitialized_variables/VarIsInitializedOp_744"
  input: "report_uninitialized_variables/VarIsInitializedOp_745"
  input: "report_uninitialized_variables/VarIsInitializedOp_746"
  input: "report_uninitialized_variables/VarIsInitializedOp_747"
  input: "report_uninitialized_variables/VarIsInitializedOp_748"
  input: "report_uninitialized_variables/VarIsInitializedOp_749"
  input: "report_uninitialized_variables/VarIsInitializedOp_750"
  input: "report_uninitialized_variables/VarIsInitializedOp_751"
  input: "report_uninitialized_variables/VarIsInitializedOp_752"
  input: "report_uninitialized_variables/VarIsInitializedOp_753"
  input: "report_uninitialized_variables/VarIsInitializedOp_754"
  input: "report_uninitialized_variables/VarIsInitializedOp_755"
  input: "report_uninitialized_variables/VarIsInitializedOp_756"
  input: "report_uninitialized_variables/VarIsInitializedOp_757"
  input: "report_uninitialized_variables/VarIsInitializedOp_758"
  input: "report_uninitialized_variables/VarIsInitializedOp_759"
  input: "report_uninitialized_variables/VarIsInitializedOp_760"
  input: "report_uninitialized_variables/VarIsInitializedOp_761"
  input: "report_uninitialized_variables/VarIsInitializedOp_762"
  input: "report_uninitialized_variables/VarIsInitializedOp_763"
  input: "report_uninitialized_variables/VarIsInitializedOp_764"
  input: "report_uninitialized_variables/VarIsInitializedOp_765"
  input: "report_uninitialized_variables/VarIsInitializedOp_766"
  input: "report_uninitialized_variables/VarIsInitializedOp_767"
  input: "report_uninitialized_variables/VarIsInitializedOp_768"
  input: "report_uninitialized_variables/VarIsInitializedOp_769"
  input: "report_uninitialized_variables/VarIsInitializedOp_770"
  input: "report_uninitialized_variables/VarIsInitializedOp_771"
  input: "report_uninitialized_variables/VarIsInitializedOp_772"
  input: "report_uninitialized_variables/VarIsInitializedOp_773"
  input: "report_uninitialized_variables/VarIsInitializedOp_774"
  input: "report_uninitialized_variables/VarIsInitializedOp_775"
  input: "report_uninitialized_variables/VarIsInitializedOp_776"
  input: "report_uninitialized_variables/VarIsInitializedOp_777"
  input: "report_uninitialized_variables/VarIsInitializedOp_778"
  input: "report_uninitialized_variables/VarIsInitializedOp_779"
  input: "report_uninitialized_variables/VarIsInitializedOp_780"
  input: "report_uninitialized_variables/VarIsInitializedOp_781"
  input: "report_uninitialized_variables/VarIsInitializedOp_782"
  input: "report_uninitialized_variables/VarIsInitializedOp_783"
  input: "report_uninitialized_variables/VarIsInitializedOp_784"
  input: "report_uninitialized_variables/VarIsInitializedOp_785"
  input: "report_uninitialized_variables/VarIsInitializedOp_786"
  input: "report_uninitialized_variables/VarIsInitializedOp_787"
  input: "report_uninitialized_variables/VarIsInitializedOp_788"
  input: "report_uninitialized_variables/VarIsInitializedOp_789"
  input: "report_uninitialized_variables/VarIsInitializedOp_790"
  input: "report_uninitialized_variables/VarIsInitializedOp_791"
  input: "report_uninitialized_variables/VarIsInitializedOp_792"
  input: "report_uninitialized_variables/VarIsInitializedOp_793"
  input: "report_uninitialized_variables/VarIsInitializedOp_794"
  input: "report_uninitialized_variables/VarIsInitializedOp_795"
  input: "report_uninitialized_variables/VarIsInitializedOp_796"
  input: "report_uninitialized_variables/VarIsInitializedOp_797"
  input: "report_uninitialized_variables/VarIsInitializedOp_798"
  input: "report_uninitialized_variables/VarIsInitializedOp_799"
  input: "report_uninitialized_variables/VarIsInitializedOp_800"
  input: "report_uninitialized_variables/VarIsInitializedOp_801"
  input: "report_uninitialized_variables/VarIsInitializedOp_802"
  input: "report_uninitialized_variables/VarIsInitializedOp_803"
  input: "report_uninitialized_variables/VarIsInitializedOp_804"
  input: "report_uninitialized_variables/VarIsInitializedOp_805"
  input: "report_uninitialized_variables/VarIsInitializedOp_806"
  input: "report_uninitialized_variables/VarIsInitializedOp_807"
  input: "report_uninitialized_variables/VarIsInitializedOp_808"
  input: "report_uninitialized_variables/VarIsInitializedOp_809"
  input: "report_uninitialized_variables/VarIsInitializedOp_810"
  input: "report_uninitialized_variables/VarIsInitializedOp_811"
  input: "report_uninitialized_variables/VarIsInitializedOp_812"
  input: "report_uninitialized_variables/VarIsInitializedOp_813"
  input: "report_uninitialized_variables/VarIsInitializedOp_814"
  input: "report_uninitialized_variables/VarIsInitializedOp_815"
  input: "report_uninitialized_variables/VarIsInitializedOp_816"
  input: "report_uninitialized_variables/VarIsInitializedOp_817"
  input: "report_uninitialized_variables/VarIsInitializedOp_818"
  input: "report_uninitialized_variables/VarIsInitializedOp_819"
  input: "report_uninitialized_variables/VarIsInitializedOp_820"
  input: "report_uninitialized_variables/VarIsInitializedOp_821"
  input: "report_uninitialized_variables/VarIsInitializedOp_822"
  input: "report_uninitialized_variables/VarIsInitializedOp_823"
  input: "report_uninitialized_variables/VarIsInitializedOp_824"
  input: "report_uninitialized_variables/VarIsInitializedOp_825"
  input: "report_uninitialized_variables/VarIsInitializedOp_826"
  input: "report_uninitialized_variables/VarIsInitializedOp_827"
  input: "report_uninitialized_variables/VarIsInitializedOp_828"
  input: "report_uninitialized_variables/VarIsInitializedOp_829"
  input: "report_uninitialized_variables/VarIsInitializedOp_830"
  input: "report_uninitialized_variables/VarIsInitializedOp_831"
  input: "report_uninitialized_variables/VarIsInitializedOp_832"
  input: "report_uninitialized_variables/VarIsInitializedOp_833"
  input: "report_uninitialized_variables/VarIsInitializedOp_834"
  input: "report_uninitialized_variables/VarIsInitializedOp_835"
  input: "report_uninitialized_variables/VarIsInitializedOp_836"
  input: "report_uninitialized_variables/VarIsInitializedOp_837"
  input: "report_uninitialized_variables/VarIsInitializedOp_838"
  input: "report_uninitialized_variables/VarIsInitializedOp_839"
  input: "report_uninitialized_variables/VarIsInitializedOp_840"
  input: "report_uninitialized_variables/VarIsInitializedOp_841"
  input: "report_uninitialized_variables/VarIsInitializedOp_842"
  input: "report_uninitialized_variables/VarIsInitializedOp_843"
  input: "report_uninitialized_variables/VarIsInitializedOp_844"
  input: "report_uninitialized_variables/VarIsInitializedOp_845"
  input: "report_uninitialized_variables/VarIsInitializedOp_846"
  input: "report_uninitialized_variables/VarIsInitializedOp_847"
  input: "report_uninitialized_variables/VarIsInitializedOp_848"
  input: "report_uninitialized_variables/VarIsInitializedOp_849"
  input: "report_uninitialized_variables/VarIsInitializedOp_850"
  input: "report_uninitialized_variables/VarIsInitializedOp_851"
  input: "report_uninitialized_variables/VarIsInitializedOp_852"
  input: "report_uninitialized_variables/VarIsInitializedOp_853"
  input: "report_uninitialized_variables/VarIsInitializedOp_854"
  input: "report_uninitialized_variables/VarIsInitializedOp_855"
  input: "report_uninitialized_variables/VarIsInitializedOp_856"
  input: "report_uninitialized_variables/VarIsInitializedOp_857"
  input: "report_uninitialized_variables/VarIsInitializedOp_858"
  input: "report_uninitialized_variables/VarIsInitializedOp_859"
  input: "report_uninitialized_variables/VarIsInitializedOp_860"
  input: "report_uninitialized_variables/VarIsInitializedOp_861"
  input: "report_uninitialized_variables/VarIsInitializedOp_862"
  input: "report_uninitialized_variables/VarIsInitializedOp_863"
  input: "report_uninitialized_variables/VarIsInitializedOp_864"
  input: "report_uninitialized_variables/VarIsInitializedOp_865"
  input: "report_uninitialized_variables/VarIsInitializedOp_866"
  input: "report_uninitialized_variables/VarIsInitializedOp_867"
  input: "report_uninitialized_variables/VarIsInitializedOp_868"
  input: "report_uninitialized_variables/VarIsInitializedOp_869"
  input: "report_uninitialized_variables/VarIsInitializedOp_870"
  input: "report_uninitialized_variables/VarIsInitializedOp_871"
  input: "report_uninitialized_variables/VarIsInitializedOp_872"
  input: "report_uninitialized_variables/VarIsInitializedOp_873"
  input: "report_uninitialized_variables/VarIsInitializedOp_874"
  input: "report_uninitialized_variables/VarIsInitializedOp_875"
  input: "report_uninitialized_variables/VarIsInitializedOp_876"
  input: "report_uninitialized_variables/VarIsInitializedOp_877"
  input: "report_uninitialized_variables/VarIsInitializedOp_878"
  input: "report_uninitialized_variables/VarIsInitializedOp_879"
  input: "report_uninitialized_variables/VarIsInitializedOp_880"
  input: "report_uninitialized_variables/VarIsInitializedOp_881"
  input: "report_uninitialized_variables/VarIsInitializedOp_882"
  input: "report_uninitialized_variables/VarIsInitializedOp_883"
  input: "report_uninitialized_variables/VarIsInitializedOp_884"
  input: "report_uninitialized_variables/VarIsInitializedOp_885"
  input: "report_uninitialized_variables/VarIsInitializedOp_886"
  input: "report_uninitialized_variables/VarIsInitializedOp_887"
  input: "report_uninitialized_variables/VarIsInitializedOp_888"
  input: "report_uninitialized_variables/VarIsInitializedOp_889"
  input: "report_uninitialized_variables/VarIsInitializedOp_890"
  input: "report_uninitialized_variables/VarIsInitializedOp_891"
  input: "report_uninitialized_variables/VarIsInitializedOp_892"
  input: "report_uninitialized_variables/VarIsInitializedOp_893"
  input: "report_uninitialized_variables/VarIsInitializedOp_894"
  input: "report_uninitialized_variables/VarIsInitializedOp_895"
  input: "report_uninitialized_variables/VarIsInitializedOp_896"
  input: "report_uninitialized_variables/VarIsInitializedOp_897"
  input: "report_uninitialized_variables/VarIsInitializedOp_898"
  input: "report_uninitialized_variables/VarIsInitializedOp_899"
  input: "report_uninitialized_variables/VarIsInitializedOp_900"
  input: "report_uninitialized_variables/VarIsInitializedOp_901"
  input: "report_uninitialized_variables/VarIsInitializedOp_902"
  input: "report_uninitialized_variables/VarIsInitializedOp_903"
  input: "report_uninitialized_variables/VarIsInitializedOp_904"
  input: "report_uninitialized_variables/VarIsInitializedOp_905"
  input: "report_uninitialized_variables/VarIsInitializedOp_906"
  input: "report_uninitialized_variables/VarIsInitializedOp_907"
  input: "report_uninitialized_variables/VarIsInitializedOp_908"
  input: "report_uninitialized_variables/VarIsInitializedOp_909"
  input: "report_uninitialized_variables/VarIsInitializedOp_910"
  input: "report_uninitialized_variables/VarIsInitializedOp_911"
  input: "report_uninitialized_variables/VarIsInitializedOp_912"
  input: "report_uninitialized_variables/VarIsInitializedOp_913"
  input: "report_uninitialized_variables/VarIsInitializedOp_914"
  input: "report_uninitialized_variables/VarIsInitializedOp_915"
  input: "report_uninitialized_variables/VarIsInitializedOp_916"
  input: "report_uninitialized_variables/VarIsInitializedOp_917"
  input: "report_uninitialized_variables/VarIsInitializedOp_918"
  input: "report_uninitialized_variables/VarIsInitializedOp_919"
  input: "report_uninitialized_variables/VarIsInitializedOp_920"
  input: "report_uninitialized_variables/VarIsInitializedOp_921"
  input: "report_uninitialized_variables/VarIsInitializedOp_922"
  input: "report_uninitialized_variables/VarIsInitializedOp_923"
  input: "report_uninitialized_variables/VarIsInitializedOp_924"
  input: "report_uninitialized_variables/VarIsInitializedOp_925"
  input: "report_uninitialized_variables/VarIsInitializedOp_926"
  input: "report_uninitialized_variables/VarIsInitializedOp_927"
  input: "report_uninitialized_variables/VarIsInitializedOp_928"
  input: "report_uninitialized_variables/VarIsInitializedOp_929"
  input: "report_uninitialized_variables/VarIsInitializedOp_930"
  input: "report_uninitialized_variables/VarIsInitializedOp_931"
  input: "report_uninitialized_variables/VarIsInitializedOp_932"
  input: "report_uninitialized_variables/VarIsInitializedOp_933"
  input: "report_uninitialized_variables/VarIsInitializedOp_934"
  input: "report_uninitialized_variables/VarIsInitializedOp_935"
  input: "report_uninitialized_variables/VarIsInitializedOp_936"
  input: "report_uninitialized_variables/VarIsInitializedOp_937"
  input: "report_uninitialized_variables/VarIsInitializedOp_938"
  input: "report_uninitialized_variables/VarIsInitializedOp_939"
  input: "report_uninitialized_variables/VarIsInitializedOp_940"
  input: "report_uninitialized_variables/VarIsInitializedOp_941"
  input: "report_uninitialized_variables/VarIsInitializedOp_942"
  input: "report_uninitialized_variables/VarIsInitializedOp_943"
  input: "report_uninitialized_variables/VarIsInitializedOp_944"
  input: "report_uninitialized_variables/VarIsInitializedOp_945"
  input: "report_uninitialized_variables/VarIsInitializedOp_946"
  input: "report_uninitialized_variables/VarIsInitializedOp_947"
  input: "report_uninitialized_variables/VarIsInitializedOp_948"
  input: "report_uninitialized_variables/VarIsInitializedOp_949"
  input: "report_uninitialized_variables/VarIsInitializedOp_950"
  input: "report_uninitialized_variables/VarIsInitializedOp_951"
  input: "report_uninitialized_variables/VarIsInitializedOp_952"
  input: "report_uninitialized_variables/VarIsInitializedOp_953"
  input: "report_uninitialized_variables/VarIsInitializedOp_954"
  input: "report_uninitialized_variables/VarIsInitializedOp_955"
  input: "report_uninitialized_variables/VarIsInitializedOp_956"
  input: "report_uninitialized_variables/VarIsInitializedOp_957"
  input: "report_uninitialized_variables/VarIsInitializedOp_958"
  input: "report_uninitialized_variables/VarIsInitializedOp_959"
  input: "report_uninitialized_variables/VarIsInitializedOp_960"
  input: "report_uninitialized_variables/VarIsInitializedOp_961"
  input: "report_uninitialized_variables/VarIsInitializedOp_962"
  input: "report_uninitialized_variables/VarIsInitializedOp_963"
  input: "report_uninitialized_variables/VarIsInitializedOp_964"
  input: "report_uninitialized_variables/VarIsInitializedOp_965"
  input: "report_uninitialized_variables/VarIsInitializedOp_966"
  input: "report_uninitialized_variables/VarIsInitializedOp_967"
  input: "report_uninitialized_variables/VarIsInitializedOp_968"
  input: "report_uninitialized_variables/VarIsInitializedOp_969"
  input: "report_uninitialized_variables/VarIsInitializedOp_970"
  input: "report_uninitialized_variables/VarIsInitializedOp_971"
  input: "report_uninitialized_variables/VarIsInitializedOp_972"
  input: "report_uninitialized_variables/VarIsInitializedOp_973"
  input: "report_uninitialized_variables/VarIsInitializedOp_974"
  input: "report_uninitialized_variables/VarIsInitializedOp_975"
  input: "report_uninitialized_variables/VarIsInitializedOp_976"
  input: "report_uninitialized_variables/VarIsInitializedOp_977"
  input: "report_uninitialized_variables/VarIsInitializedOp_978"
  input: "report_uninitialized_variables/VarIsInitializedOp_979"
  input: "report_uninitialized_variables/VarIsInitializedOp_980"
  input: "report_uninitialized_variables/VarIsInitializedOp_981"
  input: "report_uninitialized_variables/VarIsInitializedOp_982"
  input: "report_uninitialized_variables/VarIsInitializedOp_983"
  input: "report_uninitialized_variables/VarIsInitializedOp_984"
  input: "report_uninitialized_variables/VarIsInitializedOp_985"
  input: "report_uninitialized_variables/VarIsInitializedOp_986"
  input: "report_uninitialized_variables/VarIsInitializedOp_987"
  input: "report_uninitialized_variables/VarIsInitializedOp_988"
  input: "report_uninitialized_variables/VarIsInitializedOp_989"
  input: "report_uninitialized_variables/VarIsInitializedOp_990"
  input: "report_uninitialized_variables/VarIsInitializedOp_991"
  input: "report_uninitialized_variables/VarIsInitializedOp_992"
  input: "report_uninitialized_variables/VarIsInitializedOp_993"
  input: "report_uninitialized_variables/VarIsInitializedOp_994"
  input: "report_uninitialized_variables/VarIsInitializedOp_995"
  input: "report_uninitialized_variables/VarIsInitializedOp_996"
  input: "report_uninitialized_variables/VarIsInitializedOp_997"
  input: "report_uninitialized_variables/VarIsInitializedOp_998"
  input: "report_uninitialized_variables/VarIsInitializedOp_999"
  input: "report_uninitialized_variables/VarIsInitializedOp_1000"
  input: "report_uninitialized_variables/VarIsInitializedOp_1001"
  input: "report_uninitialized_variables/VarIsInitializedOp_1002"
  input: "report_uninitialized_variables/VarIsInitializedOp_1003"
  input: "report_uninitialized_variables/VarIsInitializedOp_1004"
  input: "report_uninitialized_variables/VarIsInitializedOp_1005"
  input: "report_uninitialized_variables/VarIsInitializedOp_1006"
  input: "report_uninitialized_variables/VarIsInitializedOp_1007"
  input: "report_uninitialized_variables/VarIsInitializedOp_1008"
  input: "report_uninitialized_variables/VarIsInitializedOp_1009"
  input: "report_uninitialized_variables/VarIsInitializedOp_1010"
  input: "report_uninitialized_variables/VarIsInitializedOp_1011"
  input: "report_uninitialized_variables/VarIsInitializedOp_1012"
  input: "report_uninitialized_variables/VarIsInitializedOp_1013"
  input: "report_uninitialized_variables/VarIsInitializedOp_1014"
  input: "report_uninitialized_variables/VarIsInitializedOp_1015"
  input: "report_uninitialized_variables/VarIsInitializedOp_1016"
  input: "report_uninitialized_variables/VarIsInitializedOp_1017"
  input: "report_uninitialized_variables/VarIsInitializedOp_1018"
  input: "report_uninitialized_variables/VarIsInitializedOp_1019"
  input: "report_uninitialized_variables/VarIsInitializedOp_1020"
  input: "report_uninitialized_variables/VarIsInitializedOp_1021"
  input: "report_uninitialized_variables/VarIsInitializedOp_1022"
  input: "report_uninitialized_variables/VarIsInitializedOp_1023"
  input: "report_uninitialized_variables/VarIsInitializedOp_1024"
  input: "report_uninitialized_variables/VarIsInitializedOp_1025"
  input: "report_uninitialized_variables/VarIsInitializedOp_1026"
  input: "report_uninitialized_variables/VarIsInitializedOp_1027"
  input: "report_uninitialized_variables/VarIsInitializedOp_1028"
  input: "report_uninitialized_variables/VarIsInitializedOp_1029"
  input: "report_uninitialized_variables/VarIsInitializedOp_1030"
  input: "report_uninitialized_variables/VarIsInitializedOp_1031"
  input: "report_uninitialized_variables/VarIsInitializedOp_1032"
  input: "report_uninitialized_variables/VarIsInitializedOp_1033"
  input: "report_uninitialized_variables/VarIsInitializedOp_1034"
  input: "report_uninitialized_variables/VarIsInitializedOp_1035"
  input: "report_uninitialized_variables/VarIsInitializedOp_1036"
  input: "report_uninitialized_variables/VarIsInitializedOp_1037"
  input: "report_uninitialized_variables/VarIsInitializedOp_1038"
  input: "report_uninitialized_variables/VarIsInitializedOp_1039"
  input: "report_uninitialized_variables/VarIsInitializedOp_1040"
  input: "report_uninitialized_variables/VarIsInitializedOp_1041"
  input: "report_uninitialized_variables/VarIsInitializedOp_1042"
  input: "report_uninitialized_variables/VarIsInitializedOp_1043"
  input: "report_uninitialized_variables/VarIsInitializedOp_1044"
  input: "report_uninitialized_variables/VarIsInitializedOp_1045"
  input: "report_uninitialized_variables/VarIsInitializedOp_1046"
  input: "report_uninitialized_variables/VarIsInitializedOp_1047"
  input: "report_uninitialized_variables/VarIsInitializedOp_1048"
  input: "report_uninitialized_variables/VarIsInitializedOp_1049"
  input: "report_uninitialized_variables/VarIsInitializedOp_1050"
  input: "report_uninitialized_variables/VarIsInitializedOp_1051"
  input: "report_uninitialized_variables/VarIsInitializedOp_1052"
  input: "report_uninitialized_variables/VarIsInitializedOp_1053"
  input: "report_uninitialized_variables/VarIsInitializedOp_1054"
  input: "report_uninitialized_variables/VarIsInitializedOp_1055"
  input: "report_uninitialized_variables/VarIsInitializedOp_1056"
  input: "report_uninitialized_variables/VarIsInitializedOp_1057"
  input: "report_uninitialized_variables/VarIsInitializedOp_1058"
  input: "report_uninitialized_variables/VarIsInitializedOp_1059"
  input: "report_uninitialized_variables/VarIsInitializedOp_1060"
  input: "report_uninitialized_variables/VarIsInitializedOp_1061"
  input: "report_uninitialized_variables/VarIsInitializedOp_1062"
  input: "report_uninitialized_variables/VarIsInitializedOp_1063"
  input: "report_uninitialized_variables/VarIsInitializedOp_1064"
  input: "report_uninitialized_variables/VarIsInitializedOp_1065"
  input: "report_uninitialized_variables/VarIsInitializedOp_1066"
  input: "report_uninitialized_variables/VarIsInitializedOp_1067"
  input: "report_uninitialized_variables/VarIsInitializedOp_1068"
  input: "report_uninitialized_variables/VarIsInitializedOp_1069"
  input: "report_uninitialized_variables/VarIsInitializedOp_1070"
  input: "report_uninitialized_variables/VarIsInitializedOp_1071"
  input: "report_uninitialized_variables/VarIsInitializedOp_1072"
  input: "report_uninitialized_variables/VarIsInitializedOp_1073"
  input: "report_uninitialized_variables/VarIsInitializedOp_1074"
  input: "report_uninitialized_variables/VarIsInitializedOp_1075"
  input: "report_uninitialized_variables/VarIsInitializedOp_1076"
  input: "report_uninitialized_variables/VarIsInitializedOp_1077"
  input: "report_uninitialized_variables/VarIsInitializedOp_1078"
  input: "report_uninitialized_variables/VarIsInitializedOp_1079"
  input: "report_uninitialized_variables/VarIsInitializedOp_1080"
  input: "report_uninitialized_variables/VarIsInitializedOp_1081"
  input: "report_uninitialized_variables/VarIsInitializedOp_1082"
  input: "report_uninitialized_variables/VarIsInitializedOp_1083"
  input: "report_uninitialized_variables/VarIsInitializedOp_1084"
  input: "report_uninitialized_variables/VarIsInitializedOp_1085"
  input: "report_uninitialized_variables/VarIsInitializedOp_1086"
  input: "report_uninitialized_variables/VarIsInitializedOp_1087"
  input: "report_uninitialized_variables/VarIsInitializedOp_1088"
  input: "report_uninitialized_variables/VarIsInitializedOp_1089"
  input: "report_uninitialized_variables/VarIsInitializedOp_1090"
  input: "report_uninitialized_variables/VarIsInitializedOp_1091"
  input: "report_uninitialized_variables/VarIsInitializedOp_1092"
  input: "report_uninitialized_variables/VarIsInitializedOp_1093"
  input: "report_uninitialized_variables/VarIsInitializedOp_1094"
  input: "report_uninitialized_variables/VarIsInitializedOp_1095"
  input: "report_uninitialized_variables/VarIsInitializedOp_1096"
  input: "report_uninitialized_variables/VarIsInitializedOp_1097"
  input: "report_uninitialized_variables/VarIsInitializedOp_1098"
  input: "report_uninitialized_variables/VarIsInitializedOp_1099"
  input: "report_uninitialized_variables/VarIsInitializedOp_1100"
  input: "report_uninitialized_variables/VarIsInitializedOp_1101"
  input: "report_uninitialized_variables/VarIsInitializedOp_1102"
  input: "report_uninitialized_variables/VarIsInitializedOp_1103"
  input: "report_uninitialized_variables/VarIsInitializedOp_1104"
  input: "report_uninitialized_variables/VarIsInitializedOp_1105"
  input: "report_uninitialized_variables/VarIsInitializedOp_1106"
  input: "report_uninitialized_variables/VarIsInitializedOp_1107"
  input: "report_uninitialized_variables/VarIsInitializedOp_1108"
  input: "report_uninitialized_variables/VarIsInitializedOp_1109"
  input: "report_uninitialized_variables/VarIsInitializedOp_1110"
  input: "report_uninitialized_variables/VarIsInitializedOp_1111"
  input: "report_uninitialized_variables/VarIsInitializedOp_1112"
  input: "report_uninitialized_variables/VarIsInitializedOp_1113"
  input: "report_uninitialized_variables/VarIsInitializedOp_1114"
  input: "report_uninitialized_variables/VarIsInitializedOp_1115"
  input: "report_uninitialized_variables/VarIsInitializedOp_1116"
  input: "report_uninitialized_variables/VarIsInitializedOp_1117"
  input: "report_uninitialized_variables/VarIsInitializedOp_1118"
  input: "report_uninitialized_variables/VarIsInitializedOp_1119"
  input: "report_uninitialized_variables/VarIsInitializedOp_1120"
  input: "report_uninitialized_variables/VarIsInitializedOp_1121"
  input: "report_uninitialized_variables/VarIsInitializedOp_1122"
  input: "report_uninitialized_variables/VarIsInitializedOp_1123"
  input: "report_uninitialized_variables/VarIsInitializedOp_1124"
  input: "report_uninitialized_variables/VarIsInitializedOp_1125"
  input: "report_uninitialized_variables/VarIsInitializedOp_1126"
  input: "report_uninitialized_variables/VarIsInitializedOp_1127"
  input: "report_uninitialized_variables/VarIsInitializedOp_1128"
  input: "report_uninitialized_variables/VarIsInitializedOp_1129"
  input: "report_uninitialized_variables/VarIsInitializedOp_1130"
  input: "report_uninitialized_variables/VarIsInitializedOp_1131"
  input: "report_uninitialized_variables/VarIsInitializedOp_1132"
  input: "report_uninitialized_variables/VarIsInitializedOp_1133"
  input: "report_uninitialized_variables/VarIsInitializedOp_1134"
  input: "report_uninitialized_variables/VarIsInitializedOp_1135"
  input: "report_uninitialized_variables/VarIsInitializedOp_1136"
  input: "report_uninitialized_variables/VarIsInitializedOp_1137"
  input: "report_uninitialized_variables/VarIsInitializedOp_1138"
  input: "report_uninitialized_variables/VarIsInitializedOp_1139"
  input: "report_uninitialized_variables/VarIsInitializedOp_1140"
  input: "report_uninitialized_variables/VarIsInitializedOp_1141"
  input: "report_uninitialized_variables/VarIsInitializedOp_1142"
  input: "report_uninitialized_variables/VarIsInitializedOp_1143"
  input: "report_uninitialized_variables/VarIsInitializedOp_1144"
  input: "report_uninitialized_variables/VarIsInitializedOp_1145"
  input: "report_uninitialized_variables/VarIsInitializedOp_1146"
  input: "report_uninitialized_variables/VarIsInitializedOp_1147"
  input: "report_uninitialized_variables/VarIsInitializedOp_1148"
  input: "report_uninitialized_variables/VarIsInitializedOp_1149"
  input: "report_uninitialized_variables/VarIsInitializedOp_1150"
  input: "report_uninitialized_variables/VarIsInitializedOp_1151"
  input: "report_uninitialized_variables/VarIsInitializedOp_1152"
  input: "report_uninitialized_variables/VarIsInitializedOp_1153"
  input: "report_uninitialized_variables/VarIsInitializedOp_1154"
  input: "report_uninitialized_variables/VarIsInitializedOp_1155"
  input: "report_uninitialized_variables/VarIsInitializedOp_1156"
  input: "report_uninitialized_variables/VarIsInitializedOp_1157"
  input: "report_uninitialized_variables/VarIsInitializedOp_1158"
  input: "report_uninitialized_variables/VarIsInitializedOp_1159"
  input: "report_uninitialized_variables/VarIsInitializedOp_1160"
  input: "report_uninitialized_variables/VarIsInitializedOp_1161"
  input: "report_uninitialized_variables/VarIsInitializedOp_1162"
  input: "report_uninitialized_variables/VarIsInitializedOp_1163"
  input: "report_uninitialized_variables/VarIsInitializedOp_1164"
  input: "report_uninitialized_variables/VarIsInitializedOp_1165"
  input: "report_uninitialized_variables/VarIsInitializedOp_1166"
  input: "report_uninitialized_variables/VarIsInitializedOp_1167"
  input: "report_uninitialized_variables/VarIsInitializedOp_1168"
  input: "report_uninitialized_variables/VarIsInitializedOp_1169"
  input: "report_uninitialized_variables/VarIsInitializedOp_1170"
  input: "report_uninitialized_variables/VarIsInitializedOp_1171"
  input: "report_uninitialized_variables/VarIsInitializedOp_1172"
  input: "report_uninitialized_variables/VarIsInitializedOp_1173"
  input: "report_uninitialized_variables/VarIsInitializedOp_1174"
  input: "report_uninitialized_variables/VarIsInitializedOp_1175"
  input: "report_uninitialized_variables/VarIsInitializedOp_1176"
  input: "report_uninitialized_variables/VarIsInitializedOp_1177"
  input: "report_uninitialized_variables/VarIsInitializedOp_1178"
  input: "report_uninitialized_variables/VarIsInitializedOp_1179"
  input: "report_uninitialized_variables/VarIsInitializedOp_1180"
  input: "report_uninitialized_variables/VarIsInitializedOp_1181"
  input: "report_uninitialized_variables/VarIsInitializedOp_1182"
  input: "report_uninitialized_variables/VarIsInitializedOp_1183"
  input: "report_uninitialized_variables/VarIsInitializedOp_1184"
  input: "report_uninitialized_variables/VarIsInitializedOp_1185"
  input: "report_uninitialized_variables/VarIsInitializedOp_1186"
  input: "report_uninitialized_variables/VarIsInitializedOp_1187"
  input: "report_uninitialized_variables/VarIsInitializedOp_1188"
  input: "report_uninitialized_variables/VarIsInitializedOp_1189"
  input: "report_uninitialized_variables/VarIsInitializedOp_1190"
  input: "report_uninitialized_variables/VarIsInitializedOp_1191"
  input: "report_uninitialized_variables/VarIsInitializedOp_1192"
  input: "report_uninitialized_variables/VarIsInitializedOp_1193"
  input: "report_uninitialized_variables/VarIsInitializedOp_1194"
  input: "report_uninitialized_variables/VarIsInitializedOp_1195"
  input: "report_uninitialized_variables/VarIsInitializedOp_1196"
  input: "report_uninitialized_variables/VarIsInitializedOp_1197"
  input: "report_uninitialized_variables/VarIsInitializedOp_1198"
  input: "report_uninitialized_variables/VarIsInitializedOp_1199"
  input: "report_uninitialized_variables/VarIsInitializedOp_1200"
  input: "report_uninitialized_variables/VarIsInitializedOp_1201"
  input: "report_uninitialized_variables/VarIsInitializedOp_1202"
  input: "report_uninitialized_variables/VarIsInitializedOp_1203"
  input: "report_uninitialized_variables/VarIsInitializedOp_1204"
  input: "report_uninitialized_variables/VarIsInitializedOp_1205"
  input: "report_uninitialized_variables/VarIsInitializedOp_1206"
  input: "report_uninitialized_variables/VarIsInitializedOp_1207"
  device: "/device:CPU:0"
  attr {
    key: "N"
    value {
      i: 1208
    }
  }
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1208
          }
        }
      }
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_variables/LogicalNot"
  op: "LogicalNot"
  input: "report_uninitialized_variables/stack"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1208
          }
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/Const"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1208
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
          dim {
            size: 1208
          }
        }
        string_val: "global_step"
        string_val: "electra/embeddings/word_embeddings"
        string_val: "electra/embeddings/token_type_embeddings"
        string_val: "electra/embeddings/position_embeddings"
        string_val: "electra/embeddings_1/LayerNorm/gamma"
        string_val: "electra/embeddings_1/LayerNorm/beta"
        string_val: "generator/embeddings_project/kernel"
        string_val: "generator/embeddings_project/bias"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel"
        string_val: "generator/encoder/layer_0/attention/self/query/bias"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel"
        string_val: "generator/encoder/layer_0/attention/self/key/bias"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel"
        string_val: "generator/encoder/layer_0/attention/self/value/bias"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias"
        string_val: "generator/encoder/layer_0/output/dense/kernel"
        string_val: "generator/encoder/layer_0/output/dense/bias"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel"
        string_val: "generator/encoder/layer_1/attention/self/query/bias"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel"
        string_val: "generator/encoder/layer_1/attention/self/key/bias"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel"
        string_val: "generator/encoder/layer_1/attention/self/value/bias"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias"
        string_val: "generator/encoder/layer_1/output/dense/kernel"
        string_val: "generator/encoder/layer_1/output/dense/bias"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel"
        string_val: "generator/encoder/layer_2/attention/self/query/bias"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel"
        string_val: "generator/encoder/layer_2/attention/self/key/bias"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel"
        string_val: "generator/encoder/layer_2/attention/self/value/bias"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias"
        string_val: "generator/encoder/layer_2/output/dense/kernel"
        string_val: "generator/encoder/layer_2/output/dense/bias"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel"
        string_val: "generator/encoder/layer_3/attention/self/query/bias"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel"
        string_val: "generator/encoder/layer_3/attention/self/key/bias"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel"
        string_val: "generator/encoder/layer_3/attention/self/value/bias"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias"
        string_val: "generator/encoder/layer_3/output/dense/kernel"
        string_val: "generator/encoder/layer_3/output/dense/bias"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel"
        string_val: "generator/encoder/layer_4/attention/self/query/bias"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel"
        string_val: "generator/encoder/layer_4/attention/self/key/bias"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel"
        string_val: "generator/encoder/layer_4/attention/self/value/bias"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias"
        string_val: "generator/encoder/layer_4/output/dense/kernel"
        string_val: "generator/encoder/layer_4/output/dense/bias"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel"
        string_val: "generator/encoder/layer_5/attention/self/query/bias"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel"
        string_val: "generator/encoder/layer_5/attention/self/key/bias"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel"
        string_val: "generator/encoder/layer_5/attention/self/value/bias"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias"
        string_val: "generator/encoder/layer_5/output/dense/kernel"
        string_val: "generator/encoder/layer_5/output/dense/bias"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel"
        string_val: "generator/encoder/layer_6/attention/self/query/bias"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel"
        string_val: "generator/encoder/layer_6/attention/self/key/bias"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel"
        string_val: "generator/encoder/layer_6/attention/self/value/bias"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias"
        string_val: "generator/encoder/layer_6/output/dense/kernel"
        string_val: "generator/encoder/layer_6/output/dense/bias"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel"
        string_val: "generator/encoder/layer_7/attention/self/query/bias"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel"
        string_val: "generator/encoder/layer_7/attention/self/key/bias"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel"
        string_val: "generator/encoder/layer_7/attention/self/value/bias"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias"
        string_val: "generator/encoder/layer_7/output/dense/kernel"
        string_val: "generator/encoder/layer_7/output/dense/bias"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel"
        string_val: "generator/encoder/layer_8/attention/self/query/bias"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel"
        string_val: "generator/encoder/layer_8/attention/self/key/bias"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel"
        string_val: "generator/encoder/layer_8/attention/self/value/bias"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias"
        string_val: "generator/encoder/layer_8/output/dense/kernel"
        string_val: "generator/encoder/layer_8/output/dense/bias"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel"
        string_val: "generator/encoder/layer_9/attention/self/query/bias"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel"
        string_val: "generator/encoder/layer_9/attention/self/key/bias"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel"
        string_val: "generator/encoder/layer_9/attention/self/value/bias"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias"
        string_val: "generator/encoder/layer_9/output/dense/kernel"
        string_val: "generator/encoder/layer_9/output/dense/bias"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel"
        string_val: "generator/encoder/layer_10/attention/self/query/bias"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel"
        string_val: "generator/encoder/layer_10/attention/self/key/bias"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel"
        string_val: "generator/encoder/layer_10/attention/self/value/bias"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias"
        string_val: "generator/encoder/layer_10/output/dense/kernel"
        string_val: "generator/encoder/layer_10/output/dense/bias"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel"
        string_val: "generator/encoder/layer_11/attention/self/query/bias"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel"
        string_val: "generator/encoder/layer_11/attention/self/key/bias"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel"
        string_val: "generator/encoder/layer_11/attention/self/value/bias"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias"
        string_val: "generator/encoder/layer_11/output/dense/kernel"
        string_val: "generator/encoder/layer_11/output/dense/bias"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta"
        string_val: "generator_predictions/dense/kernel"
        string_val: "generator_predictions/dense/bias"
        string_val: "generator_predictions/LayerNorm/gamma"
        string_val: "generator_predictions/LayerNorm/beta"
        string_val: "generator_predictions/output_bias"
        string_val: "electra/embeddings_3/LayerNorm/gamma"
        string_val: "electra/embeddings_3/LayerNorm/beta"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel"
        string_val: "electra/encoder/layer_0/attention/self/query/bias"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel"
        string_val: "electra/encoder/layer_0/attention/self/key/bias"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel"
        string_val: "electra/encoder/layer_0/attention/self/value/bias"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias"
        string_val: "electra/encoder/layer_0/output/dense/kernel"
        string_val: "electra/encoder/layer_0/output/dense/bias"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel"
        string_val: "electra/encoder/layer_1/attention/self/query/bias"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel"
        string_val: "electra/encoder/layer_1/attention/self/key/bias"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel"
        string_val: "electra/encoder/layer_1/attention/self/value/bias"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias"
        string_val: "electra/encoder/layer_1/output/dense/kernel"
        string_val: "electra/encoder/layer_1/output/dense/bias"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel"
        string_val: "electra/encoder/layer_2/attention/self/query/bias"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel"
        string_val: "electra/encoder/layer_2/attention/self/key/bias"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel"
        string_val: "electra/encoder/layer_2/attention/self/value/bias"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias"
        string_val: "electra/encoder/layer_2/output/dense/kernel"
        string_val: "electra/encoder/layer_2/output/dense/bias"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel"
        string_val: "electra/encoder/layer_3/attention/self/query/bias"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel"
        string_val: "electra/encoder/layer_3/attention/self/key/bias"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel"
        string_val: "electra/encoder/layer_3/attention/self/value/bias"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias"
        string_val: "electra/encoder/layer_3/output/dense/kernel"
        string_val: "electra/encoder/layer_3/output/dense/bias"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel"
        string_val: "electra/encoder/layer_4/attention/self/query/bias"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel"
        string_val: "electra/encoder/layer_4/attention/self/key/bias"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel"
        string_val: "electra/encoder/layer_4/attention/self/value/bias"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias"
        string_val: "electra/encoder/layer_4/output/dense/kernel"
        string_val: "electra/encoder/layer_4/output/dense/bias"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel"
        string_val: "electra/encoder/layer_5/attention/self/query/bias"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel"
        string_val: "electra/encoder/layer_5/attention/self/key/bias"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel"
        string_val: "electra/encoder/layer_5/attention/self/value/bias"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias"
        string_val: "electra/encoder/layer_5/output/dense/kernel"
        string_val: "electra/encoder/layer_5/output/dense/bias"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel"
        string_val: "electra/encoder/layer_6/attention/self/query/bias"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel"
        string_val: "electra/encoder/layer_6/attention/self/key/bias"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel"
        string_val: "electra/encoder/layer_6/attention/self/value/bias"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias"
        string_val: "electra/encoder/layer_6/output/dense/kernel"
        string_val: "electra/encoder/layer_6/output/dense/bias"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel"
        string_val: "electra/encoder/layer_7/attention/self/query/bias"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel"
        string_val: "electra/encoder/layer_7/attention/self/key/bias"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel"
        string_val: "electra/encoder/layer_7/attention/self/value/bias"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias"
        string_val: "electra/encoder/layer_7/output/dense/kernel"
        string_val: "electra/encoder/layer_7/output/dense/bias"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel"
        string_val: "electra/encoder/layer_8/attention/self/query/bias"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel"
        string_val: "electra/encoder/layer_8/attention/self/key/bias"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel"
        string_val: "electra/encoder/layer_8/attention/self/value/bias"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias"
        string_val: "electra/encoder/layer_8/output/dense/kernel"
        string_val: "electra/encoder/layer_8/output/dense/bias"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel"
        string_val: "electra/encoder/layer_9/attention/self/query/bias"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel"
        string_val: "electra/encoder/layer_9/attention/self/key/bias"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel"
        string_val: "electra/encoder/layer_9/attention/self/value/bias"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias"
        string_val: "electra/encoder/layer_9/output/dense/kernel"
        string_val: "electra/encoder/layer_9/output/dense/bias"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel"
        string_val: "electra/encoder/layer_10/attention/self/query/bias"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel"
        string_val: "electra/encoder/layer_10/attention/self/key/bias"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel"
        string_val: "electra/encoder/layer_10/attention/self/value/bias"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias"
        string_val: "electra/encoder/layer_10/output/dense/kernel"
        string_val: "electra/encoder/layer_10/output/dense/bias"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel"
        string_val: "electra/encoder/layer_11/attention/self/query/bias"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel"
        string_val: "electra/encoder/layer_11/attention/self/key/bias"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel"
        string_val: "electra/encoder/layer_11/attention/self/value/bias"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias"
        string_val: "electra/encoder/layer_11/output/dense/kernel"
        string_val: "electra/encoder/layer_11/output/dense/bias"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta"
        string_val: "discriminator_predictions/dense/kernel"
        string_val: "discriminator_predictions/dense/bias"
        string_val: "discriminator_predictions/dense_1/kernel"
        string_val: "discriminator_predictions/dense_1/bias"
        string_val: "electra/embeddings/word_embeddings/adam_m"
        string_val: "electra/embeddings/word_embeddings/adam_v"
        string_val: "electra/embeddings/token_type_embeddings/adam_m"
        string_val: "electra/embeddings/token_type_embeddings/adam_v"
        string_val: "electra/embeddings/position_embeddings/adam_m"
        string_val: "electra/embeddings/position_embeddings/adam_v"
        string_val: "electra/embeddings_1/LayerNorm/gamma/adam_m"
        string_val: "electra/embeddings_1/LayerNorm/gamma/adam_v"
        string_val: "electra/embeddings_1/LayerNorm/beta/adam_m"
        string_val: "electra/embeddings_1/LayerNorm/beta/adam_v"
        string_val: "generator/embeddings_project/kernel/adam_m"
        string_val: "generator/embeddings_project/kernel/adam_v"
        string_val: "generator/embeddings_project/bias/adam_m"
        string_val: "generator/embeddings_project/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_0/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
        string_val: "generator_predictions/dense/kernel/adam_m"
        string_val: "generator_predictions/dense/kernel/adam_v"
        string_val: "generator_predictions/dense/bias/adam_m"
        string_val: "generator_predictions/dense/bias/adam_v"
        string_val: "generator_predictions/LayerNorm/gamma/adam_m"
        string_val: "generator_predictions/LayerNorm/gamma/adam_v"
        string_val: "generator_predictions/LayerNorm/beta/adam_m"
        string_val: "generator_predictions/LayerNorm/beta/adam_v"
        string_val: "generator_predictions/output_bias/adam_m"
        string_val: "generator_predictions/output_bias/adam_v"
        string_val: "electra/embeddings_3/LayerNorm/gamma/adam_m"
        string_val: "electra/embeddings_3/LayerNorm/gamma/adam_v"
        string_val: "electra/embeddings_3/LayerNorm/beta/adam_m"
        string_val: "electra/embeddings_3/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_0/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
        string_val: "discriminator_predictions/dense/kernel/adam_m"
        string_val: "discriminator_predictions/dense/kernel/adam_v"
        string_val: "discriminator_predictions/dense/bias/adam_m"
        string_val: "discriminator_predictions/dense/bias/adam_v"
        string_val: "discriminator_predictions/dense_1/kernel/adam_m"
        string_val: "discriminator_predictions/dense_1/kernel/adam_v"
        string_val: "discriminator_predictions/dense_1/bias/adam_m"
        string_val: "discriminator_predictions/dense_1/bias/adam_v"
        string_val: "tpu_estimator/iterations_per_loop"
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/Shape"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1208
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice/stack"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice/stack_1"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice/stack_2"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice"
  op: "StridedSlice"
  input: "report_uninitialized_variables/boolean_mask/Shape"
  input: "report_uninitialized_variables/boolean_mask/strided_slice/stack"
  input: "report_uninitialized_variables/boolean_mask/strided_slice/stack_1"
  input: "report_uninitialized_variables/boolean_mask/strided_slice/stack_2"
  device: "/device:CPU:0"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/Prod/reduction_indices"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/Prod"
  op: "Prod"
  input: "report_uninitialized_variables/boolean_mask/strided_slice"
  input: "report_uninitialized_variables/boolean_mask/Prod/reduction_indices"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/Shape_1"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1208
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_1"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_2"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice_1"
  op: "StridedSlice"
  input: "report_uninitialized_variables/boolean_mask/Shape_1"
  input: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack"
  input: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_1"
  input: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_2"
  device: "/device:CPU:0"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 1
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/Shape_2"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1208
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_1"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_2"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/strided_slice_2"
  op: "StridedSlice"
  input: "report_uninitialized_variables/boolean_mask/Shape_2"
  input: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack"
  input: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_1"
  input: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_2"
  device: "/device:CPU:0"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 1
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/concat/values_1"
  op: "Pack"
  input: "report_uninitialized_variables/boolean_mask/Prod"
  device: "/device:CPU:0"
  attr {
    key: "N"
    value {
      i: 1
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/concat/axis"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/concat"
  op: "ConcatV2"
  input: "report_uninitialized_variables/boolean_mask/strided_slice_1"
  input: "report_uninitialized_variables/boolean_mask/concat/values_1"
  input: "report_uninitialized_variables/boolean_mask/strided_slice_2"
  input: "report_uninitialized_variables/boolean_mask/concat/axis"
  device: "/device:CPU:0"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/Reshape"
  op: "Reshape"
  input: "report_uninitialized_variables/Const"
  input: "report_uninitialized_variables/boolean_mask/concat"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1208
          }
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/Reshape_1/shape"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/Reshape_1"
  op: "Reshape"
  input: "report_uninitialized_variables/LogicalNot"
  input: "report_uninitialized_variables/boolean_mask/Reshape_1/shape"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1208
          }
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/Where"
  op: "Where"
  input: "report_uninitialized_variables/boolean_mask/Reshape_1"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/Squeeze"
  op: "Squeeze"
  input: "report_uninitialized_variables/boolean_mask/Where"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "squeeze_dims"
    value {
      list {
        i: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/GatherV2/axis"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables/boolean_mask/GatherV2"
  op: "GatherV2"
  input: "report_uninitialized_variables/boolean_mask/Reshape"
  input: "report_uninitialized_variables/boolean_mask/Squeeze"
  input: "report_uninitialized_variables/boolean_mask/GatherV2/axis"
  device: "/device:CPU:0"
  attr {
    key: "Taxis"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "Tparams"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "batch_dims"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_resources/Const"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "concat_4/axis"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "concat_4"
  op: "ConcatV2"
  input: "report_uninitialized_variables/boolean_mask/GatherV2"
  input: "report_uninitialized_resources/Const"
  input: "concat_4/axis"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "global_step"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/word_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_2"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/token_type_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_3"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/position_embeddings"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_4"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_5"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_6"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_7"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_8"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_9"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_10"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_11"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_12"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_13"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_14"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_15"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_16"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_17"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_18"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_19"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_20"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_21"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_22"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_23"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_24"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_25"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_26"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_27"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_28"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_29"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_30"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_31"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_32"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_33"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_34"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_35"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_36"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_37"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_38"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_39"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_40"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_41"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_42"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_43"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_44"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_45"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_46"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_47"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_48"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_49"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_50"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_51"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_52"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_53"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_54"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_55"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_56"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_57"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_58"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_59"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_60"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_61"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_62"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_63"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_64"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_65"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_66"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_67"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_68"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_69"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_70"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_71"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_72"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_73"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_74"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_75"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_76"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_77"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_78"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_79"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_80"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_81"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_82"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_83"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_84"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_85"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_86"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_87"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_88"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_89"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_90"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_91"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_92"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_93"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_94"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_95"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_96"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_97"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_98"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_99"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_100"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_101"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_102"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_103"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_104"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_105"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_106"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_107"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_108"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_109"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_110"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_111"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_112"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_113"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_114"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_115"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_116"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_117"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_118"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_119"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_120"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_121"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_122"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_123"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_124"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_125"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_126"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_127"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_128"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_129"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_130"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_131"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_132"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_133"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_134"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_135"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_136"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_137"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_138"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_139"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_140"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_141"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_142"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_143"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_144"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_145"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_146"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_147"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_148"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_149"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_150"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_151"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_152"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_153"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_154"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_155"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_156"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_157"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_158"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_159"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_160"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_161"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_162"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_163"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_164"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_165"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_166"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_167"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_168"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_169"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_170"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_171"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_172"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_173"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_174"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_175"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_176"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_177"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_178"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_179"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_180"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_181"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_182"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_183"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_184"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_185"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_186"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_187"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_188"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_189"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_190"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_191"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_192"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_193"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_194"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_195"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_196"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_197"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_198"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_199"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_200"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_201"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_202"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_203"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_204"
  op: "VarIsInitializedOp"
  input: "generator_predictions/output_bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_205"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_206"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_207"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_208"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_209"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_210"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_211"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_212"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_213"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_214"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_215"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_216"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_217"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_218"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_219"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_220"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_221"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_222"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_223"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_224"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_225"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_226"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_227"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_228"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_229"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_230"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_231"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_232"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_233"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_234"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_235"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_236"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_237"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_238"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_239"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_240"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_241"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_242"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_243"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_244"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_245"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_246"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_247"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_248"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_249"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_250"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_251"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_252"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_253"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_254"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_255"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_256"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_257"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_258"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_259"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_260"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_261"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_262"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_263"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_264"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_265"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_266"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_267"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_268"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_269"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_270"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_271"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_272"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_273"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_274"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_275"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_276"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_277"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_278"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_279"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_280"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_281"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_282"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_283"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_284"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_285"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_286"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_287"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_288"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_289"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_290"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_291"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_292"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_293"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_294"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_295"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_296"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_297"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_298"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_299"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_300"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_301"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_302"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_303"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_304"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_305"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_306"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_307"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_308"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_309"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_310"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_311"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_312"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_313"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_314"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_315"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_316"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_317"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_318"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_319"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_320"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_321"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_322"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_323"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_324"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_325"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_326"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_327"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_328"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_329"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_330"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_331"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_332"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_333"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_334"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_335"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_336"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_337"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_338"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_339"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_340"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_341"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_342"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_343"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_344"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_345"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_346"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_347"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_348"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_349"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_350"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_351"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_352"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_353"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_354"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_355"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_356"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_357"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_358"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_359"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_360"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_361"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_362"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_363"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_364"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_365"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_366"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_367"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_368"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_369"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_370"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_371"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_372"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_373"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_374"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_375"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_376"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_377"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_378"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_379"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_380"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_381"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_382"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_383"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_384"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_385"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_386"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_387"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_388"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_389"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_390"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_391"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_392"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_393"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_394"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_395"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_396"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_397"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_398"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_399"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_400"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_401"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/kernel"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_402"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/bias"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_403"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/word_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_404"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/word_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_405"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/token_type_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_406"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/token_type_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_407"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/position_embeddings/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_408"
  op: "VarIsInitializedOp"
  input: "electra/embeddings/position_embeddings/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_409"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_410"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_411"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_412"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_413"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_414"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_415"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_416"
  op: "VarIsInitializedOp"
  input: "generator/embeddings_project/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_417"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_418"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_419"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_420"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_421"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_422"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_423"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_424"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_425"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_426"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_427"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_428"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_429"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_430"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_431"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_432"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_433"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_434"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_435"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_436"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_437"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_438"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_439"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_440"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_441"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_442"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_443"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_444"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_445"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_446"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_447"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_448"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_449"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_450"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_451"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_452"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_453"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_454"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_455"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_456"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_457"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_458"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_459"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_460"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_461"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_462"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_463"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_464"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_465"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_466"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_467"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_468"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_469"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_470"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_471"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_472"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_473"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_474"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_475"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_476"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_477"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_478"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_479"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_480"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_481"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_482"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_483"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_484"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_485"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_486"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_487"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_488"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_489"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_490"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_491"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_492"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_493"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_494"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_495"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_496"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_497"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_498"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_499"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_500"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_501"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_502"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_503"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_504"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_505"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_506"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_507"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_508"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_509"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_510"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_511"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_512"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_513"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_514"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_515"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_516"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_517"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_518"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_519"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_520"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_521"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_522"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_523"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_524"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_525"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_526"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_527"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_528"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_529"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_530"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_531"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_532"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_533"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_534"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_535"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_536"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_537"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_538"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_539"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_540"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_541"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_542"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_543"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_544"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_545"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_546"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_547"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_548"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_549"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_550"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_551"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_552"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_553"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_554"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_555"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_556"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_557"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_558"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_559"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_560"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_561"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_562"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_563"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_564"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_565"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_566"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_567"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_568"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_569"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_570"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_571"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_572"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_573"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_574"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_575"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_576"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_577"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_578"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_579"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_580"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_581"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_582"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_583"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_584"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_585"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_586"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_587"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_588"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_589"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_590"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_591"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_592"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_593"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_594"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_595"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_596"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_597"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_598"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_599"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_600"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_601"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_602"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_603"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_604"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_605"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_606"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_607"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_608"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_609"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_610"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_611"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_612"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_613"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_614"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_615"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_616"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_617"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_618"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_619"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_620"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_621"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_622"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_623"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_624"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_625"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_626"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_627"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_628"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_629"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_630"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_631"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_632"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_633"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_634"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_635"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_636"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_637"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_638"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_639"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_640"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_641"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_642"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_643"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_644"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_645"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_646"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_647"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_648"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_649"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_650"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_651"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_652"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_653"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_654"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_655"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_656"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_657"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_658"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_659"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_660"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_661"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_662"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_663"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_664"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_665"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_666"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_667"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_668"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_669"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_670"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_671"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_672"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_673"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_674"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_675"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_676"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_677"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_678"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_679"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_680"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_681"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_682"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_683"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_684"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_685"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_686"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_687"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_688"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_689"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_690"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_691"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_692"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_693"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_694"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_695"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_696"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_697"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_698"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_699"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_700"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_701"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_702"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_703"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_704"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_705"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_706"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_707"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_708"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_709"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_710"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_711"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_712"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_713"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_714"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_715"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_716"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_717"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_718"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_719"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_720"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_721"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_722"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_723"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_724"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_725"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_726"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_727"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_728"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_729"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_730"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_731"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_732"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_733"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_734"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_735"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_736"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_737"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_738"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_739"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_740"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_741"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_742"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_743"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_744"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_745"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_746"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_747"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_748"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_749"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_750"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_751"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_752"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_753"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_754"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_755"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_756"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_757"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_758"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_759"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_760"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_761"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_762"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_763"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_764"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_765"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_766"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_767"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_768"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_769"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_770"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_771"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_772"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_773"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_774"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_775"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_776"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_777"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_778"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_779"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_780"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_781"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_782"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_783"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_784"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_785"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_786"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_787"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_788"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_789"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_790"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_791"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_792"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_793"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_794"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_795"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_796"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_797"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_798"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_799"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_800"
  op: "VarIsInitializedOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_801"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_802"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_803"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_804"
  op: "VarIsInitializedOp"
  input: "generator_predictions/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_805"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_806"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_807"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_808"
  op: "VarIsInitializedOp"
  input: "generator_predictions/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_809"
  op: "VarIsInitializedOp"
  input: "generator_predictions/output_bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_810"
  op: "VarIsInitializedOp"
  input: "generator_predictions/output_bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_811"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_812"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_813"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_814"
  op: "VarIsInitializedOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_815"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_816"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_817"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_818"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_819"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_820"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_821"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_822"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_823"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_824"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_825"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_826"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_827"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_828"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_829"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_830"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_831"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_832"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_833"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_834"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_835"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_836"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_837"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_838"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_839"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_840"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_841"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_842"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_843"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_844"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_845"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_846"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_847"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_848"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_849"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_850"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_851"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_852"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_853"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_854"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_855"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_856"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_857"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_858"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_859"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_860"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_861"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_862"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_863"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_864"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_865"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_866"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_867"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_868"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_869"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_870"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_871"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_872"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_873"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_874"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_875"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_876"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_877"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_878"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_879"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_880"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_881"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_882"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_883"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_884"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_885"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_886"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_887"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_888"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_889"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_890"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_891"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_892"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_893"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_894"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_895"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_896"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_897"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_898"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_899"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_900"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_901"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_902"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_903"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_904"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_905"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_906"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_907"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_908"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_909"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_910"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_911"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_912"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_913"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_914"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_915"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_916"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_917"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_918"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_919"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_920"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_921"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_922"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_923"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_924"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_925"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_926"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_927"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_928"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_929"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_930"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_931"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_932"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_933"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_934"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_935"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_936"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_937"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_938"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_939"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_940"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_941"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_942"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_943"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_944"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_945"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_946"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_947"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_948"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_949"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_950"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_951"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_952"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_953"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_954"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_955"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_956"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_957"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_958"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_959"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_960"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_961"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_962"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_963"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_964"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_965"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_966"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_967"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_968"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_969"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_970"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_971"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_972"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_973"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_974"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_975"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_976"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_977"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_978"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_979"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_980"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_981"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_982"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_983"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_984"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_985"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_986"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_987"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_988"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_989"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_990"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_991"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_992"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_993"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_994"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_995"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_996"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_997"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_998"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_999"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1000"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1001"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1002"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1003"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1004"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1005"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1006"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1007"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1008"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1009"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1010"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1011"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1012"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1013"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1014"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1015"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1016"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1017"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1018"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1019"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1020"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1021"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1022"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1023"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1024"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1025"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1026"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1027"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1028"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1029"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1030"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1031"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1032"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1033"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1034"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1035"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1036"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1037"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1038"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1039"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1040"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1041"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1042"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1043"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1044"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1045"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1046"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1047"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1048"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1049"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1050"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1051"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1052"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1053"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1054"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1055"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1056"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1057"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1058"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1059"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1060"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1061"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1062"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1063"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1064"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1065"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1066"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1067"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1068"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1069"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1070"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1071"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1072"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1073"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1074"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1075"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1076"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1077"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1078"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1079"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1080"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1081"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1082"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1083"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1084"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1085"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1086"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1087"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1088"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1089"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1090"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1091"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1092"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1093"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1094"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1095"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1096"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1097"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1098"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1099"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1100"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1101"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1102"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1103"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1104"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1105"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1106"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1107"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1108"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1109"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1110"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1111"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1112"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1113"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1114"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1115"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1116"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1117"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1118"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1119"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1120"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1121"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1122"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1123"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1124"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1125"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1126"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1127"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1128"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1129"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1130"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1131"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1132"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1133"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1134"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1135"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1136"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1137"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1138"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1139"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1140"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1141"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1142"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1143"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1144"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1145"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1146"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1147"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1148"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1149"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1150"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1151"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1152"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1153"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1154"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1155"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1156"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1157"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1158"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1159"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1160"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1161"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1162"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1163"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1164"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1165"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1166"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1167"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1168"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1169"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1170"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1171"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1172"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1173"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1174"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1175"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1176"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1177"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1178"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1179"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1180"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1181"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1182"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1183"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1184"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1185"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1186"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1187"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1188"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1189"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1190"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1191"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1192"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1193"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1194"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1195"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1196"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1197"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1198"
  op: "VarIsInitializedOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1199"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1200"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1201"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1202"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1203"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/kernel/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1204"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/kernel/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1205"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/bias/adam_m"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/VarIsInitializedOp_1206"
  op: "VarIsInitializedOp"
  input: "discriminator_predictions/dense_1/bias/adam_v"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/stack"
  op: "Pack"
  input: "report_uninitialized_variables_1/VarIsInitializedOp"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_2"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_3"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_4"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_5"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_6"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_7"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_8"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_9"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_10"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_11"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_12"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_13"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_14"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_15"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_16"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_17"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_18"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_19"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_20"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_21"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_22"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_23"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_24"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_25"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_26"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_27"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_28"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_29"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_30"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_31"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_32"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_33"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_34"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_35"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_36"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_37"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_38"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_39"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_40"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_41"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_42"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_43"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_44"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_45"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_46"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_47"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_48"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_49"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_50"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_51"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_52"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_53"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_54"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_55"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_56"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_57"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_58"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_59"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_60"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_61"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_62"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_63"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_64"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_65"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_66"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_67"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_68"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_69"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_70"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_71"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_72"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_73"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_74"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_75"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_76"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_77"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_78"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_79"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_80"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_81"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_82"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_83"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_84"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_85"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_86"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_87"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_88"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_89"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_90"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_91"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_92"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_93"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_94"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_95"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_96"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_97"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_98"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_99"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_100"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_101"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_102"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_103"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_104"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_105"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_106"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_107"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_108"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_109"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_110"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_111"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_112"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_113"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_114"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_115"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_116"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_117"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_118"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_119"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_120"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_121"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_122"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_123"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_124"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_125"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_126"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_127"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_128"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_129"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_130"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_131"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_132"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_133"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_134"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_135"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_136"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_137"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_138"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_139"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_140"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_141"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_142"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_143"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_144"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_145"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_146"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_147"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_148"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_149"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_150"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_151"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_152"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_153"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_154"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_155"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_156"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_157"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_158"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_159"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_160"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_161"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_162"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_163"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_164"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_165"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_166"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_167"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_168"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_169"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_170"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_171"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_172"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_173"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_174"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_175"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_176"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_177"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_178"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_179"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_180"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_181"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_182"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_183"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_184"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_185"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_186"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_187"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_188"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_189"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_190"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_191"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_192"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_193"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_194"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_195"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_196"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_197"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_198"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_199"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_200"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_201"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_202"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_203"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_204"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_205"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_206"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_207"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_208"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_209"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_210"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_211"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_212"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_213"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_214"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_215"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_216"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_217"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_218"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_219"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_220"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_221"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_222"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_223"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_224"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_225"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_226"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_227"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_228"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_229"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_230"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_231"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_232"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_233"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_234"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_235"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_236"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_237"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_238"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_239"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_240"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_241"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_242"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_243"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_244"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_245"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_246"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_247"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_248"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_249"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_250"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_251"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_252"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_253"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_254"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_255"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_256"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_257"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_258"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_259"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_260"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_261"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_262"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_263"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_264"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_265"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_266"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_267"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_268"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_269"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_270"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_271"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_272"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_273"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_274"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_275"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_276"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_277"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_278"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_279"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_280"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_281"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_282"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_283"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_284"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_285"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_286"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_287"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_288"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_289"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_290"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_291"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_292"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_293"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_294"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_295"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_296"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_297"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_298"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_299"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_300"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_301"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_302"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_303"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_304"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_305"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_306"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_307"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_308"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_309"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_310"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_311"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_312"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_313"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_314"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_315"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_316"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_317"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_318"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_319"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_320"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_321"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_322"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_323"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_324"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_325"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_326"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_327"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_328"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_329"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_330"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_331"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_332"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_333"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_334"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_335"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_336"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_337"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_338"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_339"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_340"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_341"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_342"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_343"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_344"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_345"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_346"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_347"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_348"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_349"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_350"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_351"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_352"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_353"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_354"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_355"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_356"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_357"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_358"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_359"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_360"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_361"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_362"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_363"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_364"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_365"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_366"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_367"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_368"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_369"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_370"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_371"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_372"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_373"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_374"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_375"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_376"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_377"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_378"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_379"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_380"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_381"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_382"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_383"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_384"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_385"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_386"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_387"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_388"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_389"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_390"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_391"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_392"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_393"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_394"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_395"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_396"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_397"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_398"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_399"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_400"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_401"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_402"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_403"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_404"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_405"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_406"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_407"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_408"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_409"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_410"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_411"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_412"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_413"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_414"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_415"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_416"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_417"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_418"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_419"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_420"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_421"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_422"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_423"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_424"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_425"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_426"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_427"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_428"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_429"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_430"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_431"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_432"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_433"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_434"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_435"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_436"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_437"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_438"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_439"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_440"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_441"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_442"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_443"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_444"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_445"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_446"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_447"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_448"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_449"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_450"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_451"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_452"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_453"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_454"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_455"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_456"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_457"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_458"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_459"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_460"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_461"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_462"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_463"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_464"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_465"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_466"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_467"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_468"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_469"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_470"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_471"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_472"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_473"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_474"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_475"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_476"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_477"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_478"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_479"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_480"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_481"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_482"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_483"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_484"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_485"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_486"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_487"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_488"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_489"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_490"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_491"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_492"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_493"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_494"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_495"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_496"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_497"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_498"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_499"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_500"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_501"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_502"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_503"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_504"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_505"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_506"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_507"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_508"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_509"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_510"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_511"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_512"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_513"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_514"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_515"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_516"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_517"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_518"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_519"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_520"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_521"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_522"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_523"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_524"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_525"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_526"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_527"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_528"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_529"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_530"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_531"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_532"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_533"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_534"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_535"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_536"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_537"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_538"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_539"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_540"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_541"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_542"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_543"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_544"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_545"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_546"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_547"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_548"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_549"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_550"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_551"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_552"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_553"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_554"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_555"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_556"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_557"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_558"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_559"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_560"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_561"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_562"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_563"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_564"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_565"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_566"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_567"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_568"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_569"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_570"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_571"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_572"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_573"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_574"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_575"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_576"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_577"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_578"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_579"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_580"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_581"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_582"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_583"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_584"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_585"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_586"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_587"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_588"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_589"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_590"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_591"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_592"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_593"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_594"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_595"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_596"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_597"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_598"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_599"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_600"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_601"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_602"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_603"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_604"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_605"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_606"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_607"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_608"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_609"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_610"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_611"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_612"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_613"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_614"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_615"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_616"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_617"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_618"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_619"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_620"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_621"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_622"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_623"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_624"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_625"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_626"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_627"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_628"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_629"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_630"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_631"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_632"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_633"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_634"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_635"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_636"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_637"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_638"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_639"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_640"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_641"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_642"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_643"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_644"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_645"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_646"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_647"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_648"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_649"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_650"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_651"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_652"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_653"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_654"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_655"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_656"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_657"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_658"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_659"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_660"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_661"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_662"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_663"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_664"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_665"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_666"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_667"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_668"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_669"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_670"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_671"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_672"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_673"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_674"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_675"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_676"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_677"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_678"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_679"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_680"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_681"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_682"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_683"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_684"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_685"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_686"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_687"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_688"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_689"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_690"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_691"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_692"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_693"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_694"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_695"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_696"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_697"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_698"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_699"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_700"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_701"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_702"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_703"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_704"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_705"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_706"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_707"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_708"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_709"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_710"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_711"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_712"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_713"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_714"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_715"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_716"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_717"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_718"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_719"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_720"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_721"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_722"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_723"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_724"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_725"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_726"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_727"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_728"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_729"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_730"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_731"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_732"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_733"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_734"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_735"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_736"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_737"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_738"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_739"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_740"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_741"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_742"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_743"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_744"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_745"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_746"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_747"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_748"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_749"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_750"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_751"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_752"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_753"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_754"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_755"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_756"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_757"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_758"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_759"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_760"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_761"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_762"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_763"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_764"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_765"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_766"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_767"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_768"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_769"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_770"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_771"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_772"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_773"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_774"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_775"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_776"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_777"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_778"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_779"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_780"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_781"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_782"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_783"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_784"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_785"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_786"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_787"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_788"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_789"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_790"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_791"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_792"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_793"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_794"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_795"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_796"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_797"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_798"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_799"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_800"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_801"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_802"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_803"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_804"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_805"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_806"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_807"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_808"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_809"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_810"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_811"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_812"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_813"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_814"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_815"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_816"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_817"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_818"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_819"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_820"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_821"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_822"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_823"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_824"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_825"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_826"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_827"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_828"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_829"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_830"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_831"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_832"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_833"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_834"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_835"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_836"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_837"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_838"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_839"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_840"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_841"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_842"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_843"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_844"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_845"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_846"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_847"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_848"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_849"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_850"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_851"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_852"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_853"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_854"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_855"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_856"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_857"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_858"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_859"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_860"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_861"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_862"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_863"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_864"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_865"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_866"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_867"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_868"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_869"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_870"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_871"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_872"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_873"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_874"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_875"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_876"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_877"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_878"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_879"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_880"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_881"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_882"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_883"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_884"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_885"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_886"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_887"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_888"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_889"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_890"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_891"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_892"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_893"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_894"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_895"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_896"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_897"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_898"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_899"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_900"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_901"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_902"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_903"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_904"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_905"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_906"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_907"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_908"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_909"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_910"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_911"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_912"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_913"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_914"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_915"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_916"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_917"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_918"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_919"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_920"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_921"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_922"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_923"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_924"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_925"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_926"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_927"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_928"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_929"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_930"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_931"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_932"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_933"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_934"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_935"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_936"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_937"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_938"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_939"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_940"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_941"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_942"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_943"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_944"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_945"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_946"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_947"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_948"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_949"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_950"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_951"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_952"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_953"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_954"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_955"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_956"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_957"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_958"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_959"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_960"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_961"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_962"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_963"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_964"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_965"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_966"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_967"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_968"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_969"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_970"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_971"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_972"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_973"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_974"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_975"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_976"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_977"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_978"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_979"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_980"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_981"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_982"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_983"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_984"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_985"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_986"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_987"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_988"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_989"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_990"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_991"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_992"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_993"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_994"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_995"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_996"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_997"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_998"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_999"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1000"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1001"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1002"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1003"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1004"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1005"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1006"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1007"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1008"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1009"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1010"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1011"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1012"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1013"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1014"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1015"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1016"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1017"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1018"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1019"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1020"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1021"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1022"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1023"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1024"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1025"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1026"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1027"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1028"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1029"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1030"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1031"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1032"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1033"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1034"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1035"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1036"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1037"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1038"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1039"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1040"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1041"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1042"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1043"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1044"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1045"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1046"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1047"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1048"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1049"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1050"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1051"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1052"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1053"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1054"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1055"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1056"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1057"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1058"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1059"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1060"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1061"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1062"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1063"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1064"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1065"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1066"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1067"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1068"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1069"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1070"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1071"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1072"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1073"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1074"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1075"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1076"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1077"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1078"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1079"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1080"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1081"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1082"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1083"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1084"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1085"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1086"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1087"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1088"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1089"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1090"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1091"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1092"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1093"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1094"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1095"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1096"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1097"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1098"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1099"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1100"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1101"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1102"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1103"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1104"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1105"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1106"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1107"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1108"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1109"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1110"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1111"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1112"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1113"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1114"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1115"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1116"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1117"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1118"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1119"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1120"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1121"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1122"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1123"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1124"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1125"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1126"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1127"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1128"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1129"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1130"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1131"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1132"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1133"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1134"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1135"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1136"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1137"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1138"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1139"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1140"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1141"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1142"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1143"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1144"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1145"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1146"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1147"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1148"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1149"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1150"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1151"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1152"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1153"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1154"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1155"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1156"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1157"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1158"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1159"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1160"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1161"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1162"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1163"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1164"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1165"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1166"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1167"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1168"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1169"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1170"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1171"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1172"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1173"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1174"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1175"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1176"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1177"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1178"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1179"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1180"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1181"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1182"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1183"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1184"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1185"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1186"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1187"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1188"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1189"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1190"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1191"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1192"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1193"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1194"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1195"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1196"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1197"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1198"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1199"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1200"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1201"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1202"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1203"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1204"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1205"
  input: "report_uninitialized_variables_1/VarIsInitializedOp_1206"
  device: "/device:CPU:0"
  attr {
    key: "N"
    value {
      i: 1207
    }
  }
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1207
          }
        }
      }
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_variables_1/LogicalNot"
  op: "LogicalNot"
  input: "report_uninitialized_variables_1/stack"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1207
          }
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/Const"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1207
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
          dim {
            size: 1207
          }
        }
        string_val: "global_step"
        string_val: "electra/embeddings/word_embeddings"
        string_val: "electra/embeddings/token_type_embeddings"
        string_val: "electra/embeddings/position_embeddings"
        string_val: "electra/embeddings_1/LayerNorm/gamma"
        string_val: "electra/embeddings_1/LayerNorm/beta"
        string_val: "generator/embeddings_project/kernel"
        string_val: "generator/embeddings_project/bias"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel"
        string_val: "generator/encoder/layer_0/attention/self/query/bias"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel"
        string_val: "generator/encoder/layer_0/attention/self/key/bias"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel"
        string_val: "generator/encoder/layer_0/attention/self/value/bias"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias"
        string_val: "generator/encoder/layer_0/output/dense/kernel"
        string_val: "generator/encoder/layer_0/output/dense/bias"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel"
        string_val: "generator/encoder/layer_1/attention/self/query/bias"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel"
        string_val: "generator/encoder/layer_1/attention/self/key/bias"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel"
        string_val: "generator/encoder/layer_1/attention/self/value/bias"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias"
        string_val: "generator/encoder/layer_1/output/dense/kernel"
        string_val: "generator/encoder/layer_1/output/dense/bias"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel"
        string_val: "generator/encoder/layer_2/attention/self/query/bias"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel"
        string_val: "generator/encoder/layer_2/attention/self/key/bias"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel"
        string_val: "generator/encoder/layer_2/attention/self/value/bias"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias"
        string_val: "generator/encoder/layer_2/output/dense/kernel"
        string_val: "generator/encoder/layer_2/output/dense/bias"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel"
        string_val: "generator/encoder/layer_3/attention/self/query/bias"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel"
        string_val: "generator/encoder/layer_3/attention/self/key/bias"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel"
        string_val: "generator/encoder/layer_3/attention/self/value/bias"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias"
        string_val: "generator/encoder/layer_3/output/dense/kernel"
        string_val: "generator/encoder/layer_3/output/dense/bias"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel"
        string_val: "generator/encoder/layer_4/attention/self/query/bias"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel"
        string_val: "generator/encoder/layer_4/attention/self/key/bias"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel"
        string_val: "generator/encoder/layer_4/attention/self/value/bias"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias"
        string_val: "generator/encoder/layer_4/output/dense/kernel"
        string_val: "generator/encoder/layer_4/output/dense/bias"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel"
        string_val: "generator/encoder/layer_5/attention/self/query/bias"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel"
        string_val: "generator/encoder/layer_5/attention/self/key/bias"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel"
        string_val: "generator/encoder/layer_5/attention/self/value/bias"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias"
        string_val: "generator/encoder/layer_5/output/dense/kernel"
        string_val: "generator/encoder/layer_5/output/dense/bias"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel"
        string_val: "generator/encoder/layer_6/attention/self/query/bias"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel"
        string_val: "generator/encoder/layer_6/attention/self/key/bias"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel"
        string_val: "generator/encoder/layer_6/attention/self/value/bias"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias"
        string_val: "generator/encoder/layer_6/output/dense/kernel"
        string_val: "generator/encoder/layer_6/output/dense/bias"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel"
        string_val: "generator/encoder/layer_7/attention/self/query/bias"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel"
        string_val: "generator/encoder/layer_7/attention/self/key/bias"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel"
        string_val: "generator/encoder/layer_7/attention/self/value/bias"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias"
        string_val: "generator/encoder/layer_7/output/dense/kernel"
        string_val: "generator/encoder/layer_7/output/dense/bias"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel"
        string_val: "generator/encoder/layer_8/attention/self/query/bias"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel"
        string_val: "generator/encoder/layer_8/attention/self/key/bias"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel"
        string_val: "generator/encoder/layer_8/attention/self/value/bias"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias"
        string_val: "generator/encoder/layer_8/output/dense/kernel"
        string_val: "generator/encoder/layer_8/output/dense/bias"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel"
        string_val: "generator/encoder/layer_9/attention/self/query/bias"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel"
        string_val: "generator/encoder/layer_9/attention/self/key/bias"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel"
        string_val: "generator/encoder/layer_9/attention/self/value/bias"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias"
        string_val: "generator/encoder/layer_9/output/dense/kernel"
        string_val: "generator/encoder/layer_9/output/dense/bias"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel"
        string_val: "generator/encoder/layer_10/attention/self/query/bias"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel"
        string_val: "generator/encoder/layer_10/attention/self/key/bias"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel"
        string_val: "generator/encoder/layer_10/attention/self/value/bias"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias"
        string_val: "generator/encoder/layer_10/output/dense/kernel"
        string_val: "generator/encoder/layer_10/output/dense/bias"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel"
        string_val: "generator/encoder/layer_11/attention/self/query/bias"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel"
        string_val: "generator/encoder/layer_11/attention/self/key/bias"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel"
        string_val: "generator/encoder/layer_11/attention/self/value/bias"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias"
        string_val: "generator/encoder/layer_11/output/dense/kernel"
        string_val: "generator/encoder/layer_11/output/dense/bias"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta"
        string_val: "generator_predictions/dense/kernel"
        string_val: "generator_predictions/dense/bias"
        string_val: "generator_predictions/LayerNorm/gamma"
        string_val: "generator_predictions/LayerNorm/beta"
        string_val: "generator_predictions/output_bias"
        string_val: "electra/embeddings_3/LayerNorm/gamma"
        string_val: "electra/embeddings_3/LayerNorm/beta"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel"
        string_val: "electra/encoder/layer_0/attention/self/query/bias"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel"
        string_val: "electra/encoder/layer_0/attention/self/key/bias"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel"
        string_val: "electra/encoder/layer_0/attention/self/value/bias"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias"
        string_val: "electra/encoder/layer_0/output/dense/kernel"
        string_val: "electra/encoder/layer_0/output/dense/bias"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel"
        string_val: "electra/encoder/layer_1/attention/self/query/bias"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel"
        string_val: "electra/encoder/layer_1/attention/self/key/bias"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel"
        string_val: "electra/encoder/layer_1/attention/self/value/bias"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias"
        string_val: "electra/encoder/layer_1/output/dense/kernel"
        string_val: "electra/encoder/layer_1/output/dense/bias"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel"
        string_val: "electra/encoder/layer_2/attention/self/query/bias"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel"
        string_val: "electra/encoder/layer_2/attention/self/key/bias"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel"
        string_val: "electra/encoder/layer_2/attention/self/value/bias"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias"
        string_val: "electra/encoder/layer_2/output/dense/kernel"
        string_val: "electra/encoder/layer_2/output/dense/bias"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel"
        string_val: "electra/encoder/layer_3/attention/self/query/bias"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel"
        string_val: "electra/encoder/layer_3/attention/self/key/bias"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel"
        string_val: "electra/encoder/layer_3/attention/self/value/bias"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias"
        string_val: "electra/encoder/layer_3/output/dense/kernel"
        string_val: "electra/encoder/layer_3/output/dense/bias"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel"
        string_val: "electra/encoder/layer_4/attention/self/query/bias"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel"
        string_val: "electra/encoder/layer_4/attention/self/key/bias"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel"
        string_val: "electra/encoder/layer_4/attention/self/value/bias"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias"
        string_val: "electra/encoder/layer_4/output/dense/kernel"
        string_val: "electra/encoder/layer_4/output/dense/bias"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel"
        string_val: "electra/encoder/layer_5/attention/self/query/bias"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel"
        string_val: "electra/encoder/layer_5/attention/self/key/bias"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel"
        string_val: "electra/encoder/layer_5/attention/self/value/bias"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias"
        string_val: "electra/encoder/layer_5/output/dense/kernel"
        string_val: "electra/encoder/layer_5/output/dense/bias"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel"
        string_val: "electra/encoder/layer_6/attention/self/query/bias"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel"
        string_val: "electra/encoder/layer_6/attention/self/key/bias"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel"
        string_val: "electra/encoder/layer_6/attention/self/value/bias"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias"
        string_val: "electra/encoder/layer_6/output/dense/kernel"
        string_val: "electra/encoder/layer_6/output/dense/bias"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel"
        string_val: "electra/encoder/layer_7/attention/self/query/bias"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel"
        string_val: "electra/encoder/layer_7/attention/self/key/bias"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel"
        string_val: "electra/encoder/layer_7/attention/self/value/bias"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias"
        string_val: "electra/encoder/layer_7/output/dense/kernel"
        string_val: "electra/encoder/layer_7/output/dense/bias"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel"
        string_val: "electra/encoder/layer_8/attention/self/query/bias"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel"
        string_val: "electra/encoder/layer_8/attention/self/key/bias"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel"
        string_val: "electra/encoder/layer_8/attention/self/value/bias"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias"
        string_val: "electra/encoder/layer_8/output/dense/kernel"
        string_val: "electra/encoder/layer_8/output/dense/bias"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel"
        string_val: "electra/encoder/layer_9/attention/self/query/bias"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel"
        string_val: "electra/encoder/layer_9/attention/self/key/bias"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel"
        string_val: "electra/encoder/layer_9/attention/self/value/bias"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias"
        string_val: "electra/encoder/layer_9/output/dense/kernel"
        string_val: "electra/encoder/layer_9/output/dense/bias"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel"
        string_val: "electra/encoder/layer_10/attention/self/query/bias"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel"
        string_val: "electra/encoder/layer_10/attention/self/key/bias"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel"
        string_val: "electra/encoder/layer_10/attention/self/value/bias"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias"
        string_val: "electra/encoder/layer_10/output/dense/kernel"
        string_val: "electra/encoder/layer_10/output/dense/bias"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel"
        string_val: "electra/encoder/layer_11/attention/self/query/bias"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel"
        string_val: "electra/encoder/layer_11/attention/self/key/bias"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel"
        string_val: "electra/encoder/layer_11/attention/self/value/bias"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias"
        string_val: "electra/encoder/layer_11/output/dense/kernel"
        string_val: "electra/encoder/layer_11/output/dense/bias"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta"
        string_val: "discriminator_predictions/dense/kernel"
        string_val: "discriminator_predictions/dense/bias"
        string_val: "discriminator_predictions/dense_1/kernel"
        string_val: "discriminator_predictions/dense_1/bias"
        string_val: "electra/embeddings/word_embeddings/adam_m"
        string_val: "electra/embeddings/word_embeddings/adam_v"
        string_val: "electra/embeddings/token_type_embeddings/adam_m"
        string_val: "electra/embeddings/token_type_embeddings/adam_v"
        string_val: "electra/embeddings/position_embeddings/adam_m"
        string_val: "electra/embeddings/position_embeddings/adam_v"
        string_val: "electra/embeddings_1/LayerNorm/gamma/adam_m"
        string_val: "electra/embeddings_1/LayerNorm/gamma/adam_v"
        string_val: "electra/embeddings_1/LayerNorm/beta/adam_m"
        string_val: "electra/embeddings_1/LayerNorm/beta/adam_v"
        string_val: "generator/embeddings_project/kernel/adam_m"
        string_val: "generator/embeddings_project/kernel/adam_v"
        string_val: "generator/embeddings_project/bias/adam_m"
        string_val: "generator/embeddings_project/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_0/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
        string_val: "generator_predictions/dense/kernel/adam_m"
        string_val: "generator_predictions/dense/kernel/adam_v"
        string_val: "generator_predictions/dense/bias/adam_m"
        string_val: "generator_predictions/dense/bias/adam_v"
        string_val: "generator_predictions/LayerNorm/gamma/adam_m"
        string_val: "generator_predictions/LayerNorm/gamma/adam_v"
        string_val: "generator_predictions/LayerNorm/beta/adam_m"
        string_val: "generator_predictions/LayerNorm/beta/adam_v"
        string_val: "generator_predictions/output_bias/adam_m"
        string_val: "generator_predictions/output_bias/adam_v"
        string_val: "electra/embeddings_3/LayerNorm/gamma/adam_m"
        string_val: "electra/embeddings_3/LayerNorm/gamma/adam_v"
        string_val: "electra/embeddings_3/LayerNorm/beta/adam_m"
        string_val: "electra/embeddings_3/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_0/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
        string_val: "discriminator_predictions/dense/kernel/adam_m"
        string_val: "discriminator_predictions/dense/kernel/adam_v"
        string_val: "discriminator_predictions/dense/bias/adam_m"
        string_val: "discriminator_predictions/dense/bias/adam_v"
        string_val: "discriminator_predictions/dense_1/kernel/adam_m"
        string_val: "discriminator_predictions/dense_1/kernel/adam_v"
        string_val: "discriminator_predictions/dense_1/bias/adam_m"
        string_val: "discriminator_predictions/dense_1/bias/adam_v"
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/Shape"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1207
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_1"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_2"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice"
  op: "StridedSlice"
  input: "report_uninitialized_variables_1/boolean_mask/Shape"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_1"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_2"
  device: "/device:CPU:0"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/Prod/reduction_indices"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/Prod"
  op: "Prod"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice"
  input: "report_uninitialized_variables_1/boolean_mask/Prod/reduction_indices"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/Shape_1"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1207
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_1"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_2"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1"
  op: "StridedSlice"
  input: "report_uninitialized_variables_1/boolean_mask/Shape_1"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_1"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_2"
  device: "/device:CPU:0"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 1
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/Shape_2"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1207
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_1"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_2"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2"
  op: "StridedSlice"
  input: "report_uninitialized_variables_1/boolean_mask/Shape_2"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_1"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_2"
  device: "/device:CPU:0"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "begin_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "ellipsis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "end_mask"
    value {
      i: 1
    }
  }
  attr {
    key: "new_axis_mask"
    value {
      i: 0
    }
  }
  attr {
    key: "shrink_axis_mask"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/concat/values_1"
  op: "Pack"
  input: "report_uninitialized_variables_1/boolean_mask/Prod"
  device: "/device:CPU:0"
  attr {
    key: "N"
    value {
      i: 1
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/concat/axis"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/concat"
  op: "ConcatV2"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1"
  input: "report_uninitialized_variables_1/boolean_mask/concat/values_1"
  input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2"
  input: "report_uninitialized_variables_1/boolean_mask/concat/axis"
  device: "/device:CPU:0"
  attr {
    key: "N"
    value {
      i: 3
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/Reshape"
  op: "Reshape"
  input: "report_uninitialized_variables_1/Const"
  input: "report_uninitialized_variables_1/boolean_mask/concat"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1207
          }
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/Reshape_1/shape"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/Reshape_1"
  op: "Reshape"
  input: "report_uninitialized_variables_1/LogicalNot"
  input: "report_uninitialized_variables_1/boolean_mask/Reshape_1/shape"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1207
          }
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/Where"
  op: "Where"
  input: "report_uninitialized_variables_1/boolean_mask/Reshape_1"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/Squeeze"
  op: "Squeeze"
  input: "report_uninitialized_variables_1/boolean_mask/Where"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "squeeze_dims"
    value {
      list {
        i: 1
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/GatherV2/axis"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "report_uninitialized_variables_1/boolean_mask/GatherV2"
  op: "GatherV2"
  input: "report_uninitialized_variables_1/boolean_mask/Reshape"
  input: "report_uninitialized_variables_1/boolean_mask/Squeeze"
  input: "report_uninitialized_variables_1/boolean_mask/GatherV2/axis"
  device: "/device:CPU:0"
  attr {
    key: "Taxis"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "Tparams"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
  attr {
    key: "batch_dims"
    value {
      i: 0
    }
  }
}
node {
  name: "report_uninitialized_resources_1/Const"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "concat_5/axis"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "concat_5"
  op: "ConcatV2"
  input: "report_uninitialized_variables_1/boolean_mask/GatherV2"
  input: "report_uninitialized_resources_1/Const"
  input: "concat_5/axis"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: -1
          }
        }
      }
    }
  }
}
node {
  name: "init_2"
  op: "NoOp"
  input: "^tpu_estimator/iterations_per_loop/Assign"
}
node {
  name: "init_all_tables"
  op: "NoOp"
}
node {
  name: "init_3"
  op: "NoOp"
}
node {
  name: "group_deps_5"
  op: "NoOp"
  input: "^init_2"
  input: "^init_3"
  input: "^init_all_tables"
}
node {
  name: "Merge_2/MergeSummary"
  op: "MergeSummary"
  input: "loss"
  attr {
    key: "N"
    value {
      i: 1
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "save/filename/input"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "model"
      }
    }
  }
}
node {
  name: "save/filename"
  op: "PlaceholderWithDefault"
  input: "save/filename/input"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "shape"
    value {
      shape {
      }
    }
  }
}
node {
  name: "save/Const"
  op: "PlaceholderWithDefault"
  input: "save/filename"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "shape"
    value {
      shape {
      }
    }
  }
}
node {
  name: "save/StaticRegexFullMatch"
  op: "StaticRegexFullMatch"
  input: "save/Const"
  device: "/device:CPU:*"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "pattern"
    value {
      s: "^s3://.*"
    }
  }
}
node {
  name: "save/Const_1"
  op: "Const"
  device: "/device:CPU:*"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: ".part"
      }
    }
  }
}
node {
  name: "save/Const_2"
  op: "Const"
  device: "/device:CPU:*"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "_temp/part"
      }
    }
  }
}
node {
  name: "save/Select"
  op: "Select"
  input: "save/StaticRegexFullMatch"
  input: "save/Const_1"
  input: "save/Const_2"
  device: "/device:CPU:*"
  attr {
    key: "T"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "save/StringJoin"
  op: "StringJoin"
  input: "save/Const"
  input: "save/Select"
  device: "/device:CPU:*"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "separator"
    value {
      s: ""
    }
  }
}
node {
  name: "save/num_shards"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "save/ShardedFilename/shard"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "save/ShardedFilename"
  op: "ShardedFilename"
  input: "save/StringJoin"
  input: "save/ShardedFilename/shard"
  input: "save/num_shards"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "save/SaveV2/tensor_names"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1207
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
          dim {
            size: 1207
          }
        }
        string_val: "discriminator_predictions/dense/bias"
        string_val: "discriminator_predictions/dense/bias/adam_m"
        string_val: "discriminator_predictions/dense/bias/adam_v"
        string_val: "discriminator_predictions/dense/kernel"
        string_val: "discriminator_predictions/dense/kernel/adam_m"
        string_val: "discriminator_predictions/dense/kernel/adam_v"
        string_val: "discriminator_predictions/dense_1/bias"
        string_val: "discriminator_predictions/dense_1/bias/adam_m"
        string_val: "discriminator_predictions/dense_1/bias/adam_v"
        string_val: "discriminator_predictions/dense_1/kernel"
        string_val: "discriminator_predictions/dense_1/kernel/adam_m"
        string_val: "discriminator_predictions/dense_1/kernel/adam_v"
        string_val: "electra/embeddings/position_embeddings"
        string_val: "electra/embeddings/position_embeddings/adam_m"
        string_val: "electra/embeddings/position_embeddings/adam_v"
        string_val: "electra/embeddings/token_type_embeddings"
        string_val: "electra/embeddings/token_type_embeddings/adam_m"
        string_val: "electra/embeddings/token_type_embeddings/adam_v"
        string_val: "electra/embeddings/word_embeddings"
        string_val: "electra/embeddings/word_embeddings/adam_m"
        string_val: "electra/embeddings/word_embeddings/adam_v"
        string_val: "electra/embeddings_1/LayerNorm/beta"
        string_val: "electra/embeddings_1/LayerNorm/beta/adam_m"
        string_val: "electra/embeddings_1/LayerNorm/beta/adam_v"
        string_val: "electra/embeddings_1/LayerNorm/gamma"
        string_val: "electra/embeddings_1/LayerNorm/gamma/adam_m"
        string_val: "electra/embeddings_1/LayerNorm/gamma/adam_v"
        string_val: "electra/embeddings_3/LayerNorm/beta"
        string_val: "electra/embeddings_3/LayerNorm/beta/adam_m"
        string_val: "electra/embeddings_3/LayerNorm/beta/adam_v"
        string_val: "electra/embeddings_3/LayerNorm/gamma"
        string_val: "electra/embeddings_3/LayerNorm/gamma/adam_m"
        string_val: "electra/embeddings_3/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/key/bias"
        string_val: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/query/bias"
        string_val: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/value/bias"
        string_val: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_0/output/dense/bias"
        string_val: "electra/encoder/layer_0/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/output/dense/kernel"
        string_val: "electra/encoder/layer_0/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/key/bias"
        string_val: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/query/bias"
        string_val: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/value/bias"
        string_val: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_1/output/dense/bias"
        string_val: "electra/encoder/layer_1/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/output/dense/kernel"
        string_val: "electra/encoder/layer_1/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/key/bias"
        string_val: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/query/bias"
        string_val: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/value/bias"
        string_val: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_10/output/dense/bias"
        string_val: "electra/encoder/layer_10/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/output/dense/kernel"
        string_val: "electra/encoder/layer_10/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/key/bias"
        string_val: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/query/bias"
        string_val: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/value/bias"
        string_val: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_11/output/dense/bias"
        string_val: "electra/encoder/layer_11/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/output/dense/kernel"
        string_val: "electra/encoder/layer_11/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/key/bias"
        string_val: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/query/bias"
        string_val: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/value/bias"
        string_val: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_2/output/dense/bias"
        string_val: "electra/encoder/layer_2/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/output/dense/kernel"
        string_val: "electra/encoder/layer_2/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/key/bias"
        string_val: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/query/bias"
        string_val: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/value/bias"
        string_val: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_3/output/dense/bias"
        string_val: "electra/encoder/layer_3/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/output/dense/kernel"
        string_val: "electra/encoder/layer_3/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/key/bias"
        string_val: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/query/bias"
        string_val: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/value/bias"
        string_val: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_4/output/dense/bias"
        string_val: "electra/encoder/layer_4/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/output/dense/kernel"
        string_val: "electra/encoder/layer_4/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/key/bias"
        string_val: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/query/bias"
        string_val: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/value/bias"
        string_val: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_5/output/dense/bias"
        string_val: "electra/encoder/layer_5/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/output/dense/kernel"
        string_val: "electra/encoder/layer_5/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/key/bias"
        string_val: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/query/bias"
        string_val: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/value/bias"
        string_val: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_6/output/dense/bias"
        string_val: "electra/encoder/layer_6/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/output/dense/kernel"
        string_val: "electra/encoder/layer_6/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/key/bias"
        string_val: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/query/bias"
        string_val: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/value/bias"
        string_val: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_7/output/dense/bias"
        string_val: "electra/encoder/layer_7/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/output/dense/kernel"
        string_val: "electra/encoder/layer_7/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/key/bias"
        string_val: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/query/bias"
        string_val: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/value/bias"
        string_val: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_8/output/dense/bias"
        string_val: "electra/encoder/layer_8/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/output/dense/kernel"
        string_val: "electra/encoder/layer_8/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/key/bias"
        string_val: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/query/bias"
        string_val: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/value/bias"
        string_val: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_9/output/dense/bias"
        string_val: "electra/encoder/layer_9/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/output/dense/kernel"
        string_val: "electra/encoder/layer_9/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/output/dense/kernel/adam_v"
        string_val: "generator/embeddings_project/bias"
        string_val: "generator/embeddings_project/bias/adam_m"
        string_val: "generator/embeddings_project/bias/adam_v"
        string_val: "generator/embeddings_project/kernel"
        string_val: "generator/embeddings_project/kernel/adam_m"
        string_val: "generator/embeddings_project/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/key/bias"
        string_val: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/query/bias"
        string_val: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/value/bias"
        string_val: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_0/output/dense/bias"
        string_val: "generator/encoder/layer_0/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/output/dense/kernel"
        string_val: "generator/encoder/layer_0/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/key/bias"
        string_val: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/query/bias"
        string_val: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/value/bias"
        string_val: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_1/output/dense/bias"
        string_val: "generator/encoder/layer_1/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/output/dense/kernel"
        string_val: "generator/encoder/layer_1/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/key/bias"
        string_val: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/query/bias"
        string_val: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/value/bias"
        string_val: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_10/output/dense/bias"
        string_val: "generator/encoder/layer_10/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/output/dense/kernel"
        string_val: "generator/encoder/layer_10/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/key/bias"
        string_val: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/query/bias"
        string_val: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/value/bias"
        string_val: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_11/output/dense/bias"
        string_val: "generator/encoder/layer_11/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/output/dense/kernel"
        string_val: "generator/encoder/layer_11/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/key/bias"
        string_val: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/query/bias"
        string_val: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/value/bias"
        string_val: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_2/output/dense/bias"
        string_val: "generator/encoder/layer_2/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/output/dense/kernel"
        string_val: "generator/encoder/layer_2/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/key/bias"
        string_val: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/query/bias"
        string_val: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/value/bias"
        string_val: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_3/output/dense/bias"
        string_val: "generator/encoder/layer_3/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/output/dense/kernel"
        string_val: "generator/encoder/layer_3/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/key/bias"
        string_val: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/query/bias"
        string_val: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/value/bias"
        string_val: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_4/output/dense/bias"
        string_val: "generator/encoder/layer_4/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/output/dense/kernel"
        string_val: "generator/encoder/layer_4/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/key/bias"
        string_val: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/query/bias"
        string_val: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/value/bias"
        string_val: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_5/output/dense/bias"
        string_val: "generator/encoder/layer_5/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/output/dense/kernel"
        string_val: "generator/encoder/layer_5/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/key/bias"
        string_val: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/query/bias"
        string_val: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/value/bias"
        string_val: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_6/output/dense/bias"
        string_val: "generator/encoder/layer_6/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/output/dense/kernel"
        string_val: "generator/encoder/layer_6/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/key/bias"
        string_val: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/query/bias"
        string_val: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/value/bias"
        string_val: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_7/output/dense/bias"
        string_val: "generator/encoder/layer_7/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/output/dense/kernel"
        string_val: "generator/encoder/layer_7/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/key/bias"
        string_val: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/query/bias"
        string_val: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/value/bias"
        string_val: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_8/output/dense/bias"
        string_val: "generator/encoder/layer_8/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/output/dense/kernel"
        string_val: "generator/encoder/layer_8/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/key/bias"
        string_val: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/query/bias"
        string_val: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/value/bias"
        string_val: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_9/output/dense/bias"
        string_val: "generator/encoder/layer_9/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/output/dense/kernel"
        string_val: "generator/encoder/layer_9/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/output/dense/kernel/adam_v"
        string_val: "generator_predictions/LayerNorm/beta"
        string_val: "generator_predictions/LayerNorm/beta/adam_m"
        string_val: "generator_predictions/LayerNorm/beta/adam_v"
        string_val: "generator_predictions/LayerNorm/gamma"
        string_val: "generator_predictions/LayerNorm/gamma/adam_m"
        string_val: "generator_predictions/LayerNorm/gamma/adam_v"
        string_val: "generator_predictions/dense/bias"
        string_val: "generator_predictions/dense/bias/adam_m"
        string_val: "generator_predictions/dense/bias/adam_v"
        string_val: "generator_predictions/dense/kernel"
        string_val: "generator_predictions/dense/kernel/adam_m"
        string_val: "generator_predictions/dense/kernel/adam_v"
        string_val: "generator_predictions/output_bias"
        string_val: "generator_predictions/output_bias/adam_m"
        string_val: "generator_predictions/output_bias/adam_v"
        string_val: "global_step"
      }
    }
  }
}
node {
  name: "save/SaveV2/shape_and_slices"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1207
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
          dim {
            size: 1207
          }
        }
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
      }
    }
  }
}
node {
  name: "save/SaveV2"
  op: "SaveV2"
  input: "save/ShardedFilename"
  input: "save/SaveV2/tensor_names"
  input: "save/SaveV2/shape_and_slices"
  input: "discriminator_predictions/dense/bias/Read/ReadVariableOp"
  input: "discriminator_predictions/dense/bias/adam_m/Read/ReadVariableOp"
  input: "discriminator_predictions/dense/bias/adam_v/Read/ReadVariableOp"
  input: "discriminator_predictions/dense/kernel/Read/ReadVariableOp"
  input: "discriminator_predictions/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "discriminator_predictions/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "discriminator_predictions/dense_1/bias/Read/ReadVariableOp"
  input: "discriminator_predictions/dense_1/bias/adam_m/Read/ReadVariableOp"
  input: "discriminator_predictions/dense_1/bias/adam_v/Read/ReadVariableOp"
  input: "discriminator_predictions/dense_1/kernel/Read/ReadVariableOp"
  input: "discriminator_predictions/dense_1/kernel/adam_m/Read/ReadVariableOp"
  input: "discriminator_predictions/dense_1/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/embeddings/position_embeddings/Read/ReadVariableOp"
  input: "electra/embeddings/position_embeddings/adam_m/Read/ReadVariableOp"
  input: "electra/embeddings/position_embeddings/adam_v/Read/ReadVariableOp"
  input: "electra/embeddings/token_type_embeddings/Read/ReadVariableOp"
  input: "electra/embeddings/token_type_embeddings/adam_m/Read/ReadVariableOp"
  input: "electra/embeddings/token_type_embeddings/adam_v/Read/ReadVariableOp"
  input: "electra/embeddings/word_embeddings/Read/ReadVariableOp"
  input: "electra/embeddings/word_embeddings/adam_m/Read/ReadVariableOp"
  input: "electra/embeddings/word_embeddings/adam_v/Read/ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/embeddings_project/bias/Read/ReadVariableOp"
  input: "generator/embeddings_project/bias/adam_m/Read/ReadVariableOp"
  input: "generator/embeddings_project/bias/adam_v/Read/ReadVariableOp"
  input: "generator/embeddings_project/kernel/Read/ReadVariableOp"
  input: "generator/embeddings_project/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/embeddings_project/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator_predictions/LayerNorm/beta/Read/ReadVariableOp"
  input: "generator_predictions/LayerNorm/beta/adam_m/Read/ReadVariableOp"
  input: "generator_predictions/LayerNorm/beta/adam_v/Read/ReadVariableOp"
  input: "generator_predictions/LayerNorm/gamma/Read/ReadVariableOp"
  input: "generator_predictions/LayerNorm/gamma/adam_m/Read/ReadVariableOp"
  input: "generator_predictions/LayerNorm/gamma/adam_v/Read/ReadVariableOp"
  input: "generator_predictions/dense/bias/Read/ReadVariableOp"
  input: "generator_predictions/dense/bias/adam_m/Read/ReadVariableOp"
  input: "generator_predictions/dense/bias/adam_v/Read/ReadVariableOp"
  input: "generator_predictions/dense/kernel/Read/ReadVariableOp"
  input: "generator_predictions/dense/kernel/adam_m/Read/ReadVariableOp"
  input: "generator_predictions/dense/kernel/adam_v/Read/ReadVariableOp"
  input: "generator_predictions/output_bias/Read/ReadVariableOp"
  input: "generator_predictions/output_bias/adam_m/Read/ReadVariableOp"
  input: "generator_predictions/output_bias/adam_v/Read/ReadVariableOp"
  input: "global_step/Read/ReadVariableOp"
  device: "/device:CPU:0"
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_INT64
      }
    }
  }
}
node {
  name: "save/control_dependency"
  op: "Identity"
  input: "save/ShardedFilename"
  input: "^save/SaveV2"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@save/ShardedFilename"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "save/MergeV2Checkpoints/checkpoint_prefixes"
  op: "Pack"
  input: "save/ShardedFilename"
  input: "^save/control_dependency"
  device: "/device:CPU:0"
  attr {
    key: "N"
    value {
      i: 1
    }
  }
  attr {
    key: "T"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "axis"
    value {
      i: 0
    }
  }
}
node {
  name: "save/MergeV2Checkpoints"
  op: "MergeV2Checkpoints"
  input: "save/MergeV2Checkpoints/checkpoint_prefixes"
  input: "save/Const"
  device: "/device:CPU:0"
  attr {
    key: "delete_old_dirs"
    value {
      b: true
    }
  }
}
node {
  name: "save/Identity"
  op: "Identity"
  input: "save/Const"
  input: "^save/MergeV2Checkpoints"
  input: "^save/control_dependency"
  device: "/device:CPU:0"
  attr {
    key: "T"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "save/RestoreV2/tensor_names"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1207
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
          dim {
            size: 1207
          }
        }
        string_val: "discriminator_predictions/dense/bias"
        string_val: "discriminator_predictions/dense/bias/adam_m"
        string_val: "discriminator_predictions/dense/bias/adam_v"
        string_val: "discriminator_predictions/dense/kernel"
        string_val: "discriminator_predictions/dense/kernel/adam_m"
        string_val: "discriminator_predictions/dense/kernel/adam_v"
        string_val: "discriminator_predictions/dense_1/bias"
        string_val: "discriminator_predictions/dense_1/bias/adam_m"
        string_val: "discriminator_predictions/dense_1/bias/adam_v"
        string_val: "discriminator_predictions/dense_1/kernel"
        string_val: "discriminator_predictions/dense_1/kernel/adam_m"
        string_val: "discriminator_predictions/dense_1/kernel/adam_v"
        string_val: "electra/embeddings/position_embeddings"
        string_val: "electra/embeddings/position_embeddings/adam_m"
        string_val: "electra/embeddings/position_embeddings/adam_v"
        string_val: "electra/embeddings/token_type_embeddings"
        string_val: "electra/embeddings/token_type_embeddings/adam_m"
        string_val: "electra/embeddings/token_type_embeddings/adam_v"
        string_val: "electra/embeddings/word_embeddings"
        string_val: "electra/embeddings/word_embeddings/adam_m"
        string_val: "electra/embeddings/word_embeddings/adam_v"
        string_val: "electra/embeddings_1/LayerNorm/beta"
        string_val: "electra/embeddings_1/LayerNorm/beta/adam_m"
        string_val: "electra/embeddings_1/LayerNorm/beta/adam_v"
        string_val: "electra/embeddings_1/LayerNorm/gamma"
        string_val: "electra/embeddings_1/LayerNorm/gamma/adam_m"
        string_val: "electra/embeddings_1/LayerNorm/gamma/adam_v"
        string_val: "electra/embeddings_3/LayerNorm/beta"
        string_val: "electra/embeddings_3/LayerNorm/beta/adam_m"
        string_val: "electra/embeddings_3/LayerNorm/beta/adam_v"
        string_val: "electra/embeddings_3/LayerNorm/gamma"
        string_val: "electra/embeddings_3/LayerNorm/gamma/adam_m"
        string_val: "electra/embeddings_3/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/key/bias"
        string_val: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/query/bias"
        string_val: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/value/bias"
        string_val: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_0/output/dense/bias"
        string_val: "electra/encoder/layer_0/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_0/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_0/output/dense/kernel"
        string_val: "electra/encoder/layer_0/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_0/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/key/bias"
        string_val: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/query/bias"
        string_val: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/value/bias"
        string_val: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_1/output/dense/bias"
        string_val: "electra/encoder/layer_1/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_1/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_1/output/dense/kernel"
        string_val: "electra/encoder/layer_1/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_1/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/key/bias"
        string_val: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/query/bias"
        string_val: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/value/bias"
        string_val: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_10/output/dense/bias"
        string_val: "electra/encoder/layer_10/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_10/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_10/output/dense/kernel"
        string_val: "electra/encoder/layer_10/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_10/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/key/bias"
        string_val: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/query/bias"
        string_val: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/value/bias"
        string_val: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_11/output/dense/bias"
        string_val: "electra/encoder/layer_11/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_11/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_11/output/dense/kernel"
        string_val: "electra/encoder/layer_11/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_11/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/key/bias"
        string_val: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/query/bias"
        string_val: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/value/bias"
        string_val: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_2/output/dense/bias"
        string_val: "electra/encoder/layer_2/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_2/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_2/output/dense/kernel"
        string_val: "electra/encoder/layer_2/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_2/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/key/bias"
        string_val: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/query/bias"
        string_val: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/value/bias"
        string_val: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_3/output/dense/bias"
        string_val: "electra/encoder/layer_3/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_3/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_3/output/dense/kernel"
        string_val: "electra/encoder/layer_3/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_3/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/key/bias"
        string_val: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/query/bias"
        string_val: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/value/bias"
        string_val: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_4/output/dense/bias"
        string_val: "electra/encoder/layer_4/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_4/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_4/output/dense/kernel"
        string_val: "electra/encoder/layer_4/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_4/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/key/bias"
        string_val: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/query/bias"
        string_val: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/value/bias"
        string_val: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_5/output/dense/bias"
        string_val: "electra/encoder/layer_5/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_5/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_5/output/dense/kernel"
        string_val: "electra/encoder/layer_5/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_5/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/key/bias"
        string_val: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/query/bias"
        string_val: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/value/bias"
        string_val: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_6/output/dense/bias"
        string_val: "electra/encoder/layer_6/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_6/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_6/output/dense/kernel"
        string_val: "electra/encoder/layer_6/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_6/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/key/bias"
        string_val: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/query/bias"
        string_val: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/value/bias"
        string_val: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_7/output/dense/bias"
        string_val: "electra/encoder/layer_7/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_7/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_7/output/dense/kernel"
        string_val: "electra/encoder/layer_7/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_7/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/key/bias"
        string_val: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/query/bias"
        string_val: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/value/bias"
        string_val: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_8/output/dense/bias"
        string_val: "electra/encoder/layer_8/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_8/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_8/output/dense/kernel"
        string_val: "electra/encoder/layer_8/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_8/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/key/bias"
        string_val: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/query/bias"
        string_val: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/value/bias"
        string_val: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
        string_val: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
        string_val: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
        string_val: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
        string_val: "electra/encoder/layer_9/output/dense/bias"
        string_val: "electra/encoder/layer_9/output/dense/bias/adam_m"
        string_val: "electra/encoder/layer_9/output/dense/bias/adam_v"
        string_val: "electra/encoder/layer_9/output/dense/kernel"
        string_val: "electra/encoder/layer_9/output/dense/kernel/adam_m"
        string_val: "electra/encoder/layer_9/output/dense/kernel/adam_v"
        string_val: "generator/embeddings_project/bias"
        string_val: "generator/embeddings_project/bias/adam_m"
        string_val: "generator/embeddings_project/bias/adam_v"
        string_val: "generator/embeddings_project/kernel"
        string_val: "generator/embeddings_project/kernel/adam_m"
        string_val: "generator/embeddings_project/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/key/bias"
        string_val: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/query/bias"
        string_val: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/value/bias"
        string_val: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_0/output/dense/bias"
        string_val: "generator/encoder/layer_0/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_0/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_0/output/dense/kernel"
        string_val: "generator/encoder/layer_0/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_0/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/key/bias"
        string_val: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/query/bias"
        string_val: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/value/bias"
        string_val: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_1/output/dense/bias"
        string_val: "generator/encoder/layer_1/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_1/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_1/output/dense/kernel"
        string_val: "generator/encoder/layer_1/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_1/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/key/bias"
        string_val: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/query/bias"
        string_val: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/value/bias"
        string_val: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_10/output/dense/bias"
        string_val: "generator/encoder/layer_10/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_10/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_10/output/dense/kernel"
        string_val: "generator/encoder/layer_10/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_10/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/key/bias"
        string_val: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/query/bias"
        string_val: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/value/bias"
        string_val: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_11/output/dense/bias"
        string_val: "generator/encoder/layer_11/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_11/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_11/output/dense/kernel"
        string_val: "generator/encoder/layer_11/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_11/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/key/bias"
        string_val: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/query/bias"
        string_val: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/value/bias"
        string_val: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_2/output/dense/bias"
        string_val: "generator/encoder/layer_2/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_2/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_2/output/dense/kernel"
        string_val: "generator/encoder/layer_2/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_2/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/key/bias"
        string_val: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/query/bias"
        string_val: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/value/bias"
        string_val: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_3/output/dense/bias"
        string_val: "generator/encoder/layer_3/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_3/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_3/output/dense/kernel"
        string_val: "generator/encoder/layer_3/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_3/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/key/bias"
        string_val: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/query/bias"
        string_val: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/value/bias"
        string_val: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_4/output/dense/bias"
        string_val: "generator/encoder/layer_4/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_4/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_4/output/dense/kernel"
        string_val: "generator/encoder/layer_4/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_4/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/key/bias"
        string_val: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/query/bias"
        string_val: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/value/bias"
        string_val: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_5/output/dense/bias"
        string_val: "generator/encoder/layer_5/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_5/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_5/output/dense/kernel"
        string_val: "generator/encoder/layer_5/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_5/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/key/bias"
        string_val: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/query/bias"
        string_val: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/value/bias"
        string_val: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_6/output/dense/bias"
        string_val: "generator/encoder/layer_6/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_6/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_6/output/dense/kernel"
        string_val: "generator/encoder/layer_6/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_6/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/key/bias"
        string_val: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/query/bias"
        string_val: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/value/bias"
        string_val: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_7/output/dense/bias"
        string_val: "generator/encoder/layer_7/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_7/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_7/output/dense/kernel"
        string_val: "generator/encoder/layer_7/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_7/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/key/bias"
        string_val: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/query/bias"
        string_val: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/value/bias"
        string_val: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_8/output/dense/bias"
        string_val: "generator/encoder/layer_8/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_8/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_8/output/dense/kernel"
        string_val: "generator/encoder/layer_8/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_8/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/key/bias"
        string_val: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/query/bias"
        string_val: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/value/bias"
        string_val: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
        string_val: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
        string_val: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
        string_val: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
        string_val: "generator/encoder/layer_9/output/dense/bias"
        string_val: "generator/encoder/layer_9/output/dense/bias/adam_m"
        string_val: "generator/encoder/layer_9/output/dense/bias/adam_v"
        string_val: "generator/encoder/layer_9/output/dense/kernel"
        string_val: "generator/encoder/layer_9/output/dense/kernel/adam_m"
        string_val: "generator/encoder/layer_9/output/dense/kernel/adam_v"
        string_val: "generator_predictions/LayerNorm/beta"
        string_val: "generator_predictions/LayerNorm/beta/adam_m"
        string_val: "generator_predictions/LayerNorm/beta/adam_v"
        string_val: "generator_predictions/LayerNorm/gamma"
        string_val: "generator_predictions/LayerNorm/gamma/adam_m"
        string_val: "generator_predictions/LayerNorm/gamma/adam_v"
        string_val: "generator_predictions/dense/bias"
        string_val: "generator_predictions/dense/bias/adam_m"
        string_val: "generator_predictions/dense/bias/adam_v"
        string_val: "generator_predictions/dense/kernel"
        string_val: "generator_predictions/dense/kernel/adam_m"
        string_val: "generator_predictions/dense/kernel/adam_v"
        string_val: "generator_predictions/output_bias"
        string_val: "generator_predictions/output_bias/adam_m"
        string_val: "generator_predictions/output_bias/adam_v"
        string_val: "global_step"
      }
    }
  }
}
node {
  name: "save/RestoreV2/shape_and_slices"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1207
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
          dim {
            size: 1207
          }
        }
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
        string_val: ""
      }
    }
  }
}
node {
  name: "save/RestoreV2"
  op: "RestoreV2"
  input: "save/Const"
  input: "save/RestoreV2/tensor_names"
  input: "save/RestoreV2/shape_and_slices"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
        shape {
          unknown_rank: true
        }
      }
    }
  }
  attr {
    key: "dtypes"
    value {
      list {
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_FLOAT
        type: DT_INT64
      }
    }
  }
}
node {
  name: "save/Identity_1"
  op: "Identity"
  input: "save/RestoreV2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/bias"
  input: "save/Identity_1"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_2"
  op: "Identity"
  input: "save/RestoreV2:1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/bias/adam_m"
  input: "save/Identity_2"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_3"
  op: "Identity"
  input: "save/RestoreV2:2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_2"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/bias/adam_v"
  input: "save/Identity_3"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_4"
  op: "Identity"
  input: "save/RestoreV2:3"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_3"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/kernel"
  input: "save/Identity_4"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_5"
  op: "Identity"
  input: "save/RestoreV2:4"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_4"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/kernel/adam_m"
  input: "save/Identity_5"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_6"
  op: "Identity"
  input: "save/RestoreV2:5"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_5"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense/kernel/adam_v"
  input: "save/Identity_6"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_7"
  op: "Identity"
  input: "save/RestoreV2:6"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_6"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/bias"
  input: "save/Identity_7"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_8"
  op: "Identity"
  input: "save/RestoreV2:7"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_7"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/bias/adam_m"
  input: "save/Identity_8"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_9"
  op: "Identity"
  input: "save/RestoreV2:8"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_8"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/bias/adam_v"
  input: "save/Identity_9"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_10"
  op: "Identity"
  input: "save/RestoreV2:9"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_9"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/kernel"
  input: "save/Identity_10"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_11"
  op: "Identity"
  input: "save/RestoreV2:10"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_10"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/kernel/adam_m"
  input: "save/Identity_11"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_12"
  op: "Identity"
  input: "save/RestoreV2:11"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_11"
  op: "AssignVariableOp"
  input: "discriminator_predictions/dense_1/kernel/adam_v"
  input: "save/Identity_12"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_13"
  op: "Identity"
  input: "save/RestoreV2:12"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_12"
  op: "AssignVariableOp"
  input: "electra/embeddings/position_embeddings"
  input: "save/Identity_13"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_14"
  op: "Identity"
  input: "save/RestoreV2:13"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_13"
  op: "AssignVariableOp"
  input: "electra/embeddings/position_embeddings/adam_m"
  input: "save/Identity_14"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_15"
  op: "Identity"
  input: "save/RestoreV2:14"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_14"
  op: "AssignVariableOp"
  input: "electra/embeddings/position_embeddings/adam_v"
  input: "save/Identity_15"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_16"
  op: "Identity"
  input: "save/RestoreV2:15"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_15"
  op: "AssignVariableOp"
  input: "electra/embeddings/token_type_embeddings"
  input: "save/Identity_16"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_17"
  op: "Identity"
  input: "save/RestoreV2:16"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_16"
  op: "AssignVariableOp"
  input: "electra/embeddings/token_type_embeddings/adam_m"
  input: "save/Identity_17"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_18"
  op: "Identity"
  input: "save/RestoreV2:17"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_17"
  op: "AssignVariableOp"
  input: "electra/embeddings/token_type_embeddings/adam_v"
  input: "save/Identity_18"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_19"
  op: "Identity"
  input: "save/RestoreV2:18"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_18"
  op: "AssignVariableOp"
  input: "electra/embeddings/word_embeddings"
  input: "save/Identity_19"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_20"
  op: "Identity"
  input: "save/RestoreV2:19"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_19"
  op: "AssignVariableOp"
  input: "electra/embeddings/word_embeddings/adam_m"
  input: "save/Identity_20"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_21"
  op: "Identity"
  input: "save/RestoreV2:20"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_20"
  op: "AssignVariableOp"
  input: "electra/embeddings/word_embeddings/adam_v"
  input: "save/Identity_21"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_22"
  op: "Identity"
  input: "save/RestoreV2:21"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_21"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta"
  input: "save/Identity_22"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_23"
  op: "Identity"
  input: "save/RestoreV2:22"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_22"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_m"
  input: "save/Identity_23"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_24"
  op: "Identity"
  input: "save/RestoreV2:23"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_23"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/beta/adam_v"
  input: "save/Identity_24"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_25"
  op: "Identity"
  input: "save/RestoreV2:24"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_24"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma"
  input: "save/Identity_25"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_26"
  op: "Identity"
  input: "save/RestoreV2:25"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_25"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_m"
  input: "save/Identity_26"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_27"
  op: "Identity"
  input: "save/RestoreV2:26"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_26"
  op: "AssignVariableOp"
  input: "electra/embeddings_1/LayerNorm/gamma/adam_v"
  input: "save/Identity_27"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_28"
  op: "Identity"
  input: "save/RestoreV2:27"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_27"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta"
  input: "save/Identity_28"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_29"
  op: "Identity"
  input: "save/RestoreV2:28"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_28"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_m"
  input: "save/Identity_29"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_30"
  op: "Identity"
  input: "save/RestoreV2:29"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_29"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/beta/adam_v"
  input: "save/Identity_30"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_31"
  op: "Identity"
  input: "save/RestoreV2:30"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_30"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma"
  input: "save/Identity_31"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_32"
  op: "Identity"
  input: "save/RestoreV2:31"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_31"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_m"
  input: "save/Identity_32"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_33"
  op: "Identity"
  input: "save/RestoreV2:32"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_32"
  op: "AssignVariableOp"
  input: "electra/embeddings_3/LayerNorm/gamma/adam_v"
  input: "save/Identity_33"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_34"
  op: "Identity"
  input: "save/RestoreV2:33"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_33"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta"
  input: "save/Identity_34"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_35"
  op: "Identity"
  input: "save/RestoreV2:34"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_34"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_35"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_36"
  op: "Identity"
  input: "save/RestoreV2:35"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_35"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_36"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_37"
  op: "Identity"
  input: "save/RestoreV2:36"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_36"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma"
  input: "save/Identity_37"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_38"
  op: "Identity"
  input: "save/RestoreV2:37"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_37"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_38"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_39"
  op: "Identity"
  input: "save/RestoreV2:38"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_38"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_39"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_40"
  op: "Identity"
  input: "save/RestoreV2:39"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_39"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias"
  input: "save/Identity_40"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_41"
  op: "Identity"
  input: "save/RestoreV2:40"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_40"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_m"
  input: "save/Identity_41"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_42"
  op: "Identity"
  input: "save/RestoreV2:41"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_41"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/bias/adam_v"
  input: "save/Identity_42"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_43"
  op: "Identity"
  input: "save/RestoreV2:42"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_42"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel"
  input: "save/Identity_43"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_44"
  op: "Identity"
  input: "save/RestoreV2:43"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_43"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_m"
  input: "save/Identity_44"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_45"
  op: "Identity"
  input: "save/RestoreV2:44"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_44"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/output/dense/kernel/adam_v"
  input: "save/Identity_45"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_46"
  op: "Identity"
  input: "save/RestoreV2:45"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_45"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias"
  input: "save/Identity_46"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_47"
  op: "Identity"
  input: "save/RestoreV2:46"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_46"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_m"
  input: "save/Identity_47"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_48"
  op: "Identity"
  input: "save/RestoreV2:47"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_47"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/bias/adam_v"
  input: "save/Identity_48"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_49"
  op: "Identity"
  input: "save/RestoreV2:48"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_48"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel"
  input: "save/Identity_49"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_50"
  op: "Identity"
  input: "save/RestoreV2:49"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_49"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_m"
  input: "save/Identity_50"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_51"
  op: "Identity"
  input: "save/RestoreV2:50"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_50"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/key/kernel/adam_v"
  input: "save/Identity_51"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_52"
  op: "Identity"
  input: "save/RestoreV2:51"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_51"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias"
  input: "save/Identity_52"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_53"
  op: "Identity"
  input: "save/RestoreV2:52"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_52"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_m"
  input: "save/Identity_53"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_54"
  op: "Identity"
  input: "save/RestoreV2:53"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_53"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/bias/adam_v"
  input: "save/Identity_54"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_55"
  op: "Identity"
  input: "save/RestoreV2:54"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_54"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel"
  input: "save/Identity_55"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_56"
  op: "Identity"
  input: "save/RestoreV2:55"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_55"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_m"
  input: "save/Identity_56"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_57"
  op: "Identity"
  input: "save/RestoreV2:56"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_56"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/query/kernel/adam_v"
  input: "save/Identity_57"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_58"
  op: "Identity"
  input: "save/RestoreV2:57"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_57"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias"
  input: "save/Identity_58"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_59"
  op: "Identity"
  input: "save/RestoreV2:58"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_58"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_m"
  input: "save/Identity_59"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_60"
  op: "Identity"
  input: "save/RestoreV2:59"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_59"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/bias/adam_v"
  input: "save/Identity_60"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_61"
  op: "Identity"
  input: "save/RestoreV2:60"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_60"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel"
  input: "save/Identity_61"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_62"
  op: "Identity"
  input: "save/RestoreV2:61"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_61"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_m"
  input: "save/Identity_62"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_63"
  op: "Identity"
  input: "save/RestoreV2:62"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_62"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/attention/self/value/kernel/adam_v"
  input: "save/Identity_63"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_64"
  op: "Identity"
  input: "save/RestoreV2:63"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_63"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias"
  input: "save/Identity_64"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_65"
  op: "Identity"
  input: "save/RestoreV2:64"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_64"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_m"
  input: "save/Identity_65"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_66"
  op: "Identity"
  input: "save/RestoreV2:65"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_65"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/bias/adam_v"
  input: "save/Identity_66"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_67"
  op: "Identity"
  input: "save/RestoreV2:66"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_66"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel"
  input: "save/Identity_67"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_68"
  op: "Identity"
  input: "save/RestoreV2:67"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_67"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_m"
  input: "save/Identity_68"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_69"
  op: "Identity"
  input: "save/RestoreV2:68"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_68"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/intermediate/dense/kernel/adam_v"
  input: "save/Identity_69"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_70"
  op: "Identity"
  input: "save/RestoreV2:69"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_69"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta"
  input: "save/Identity_70"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_71"
  op: "Identity"
  input: "save/RestoreV2:70"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_70"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_m"
  input: "save/Identity_71"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_72"
  op: "Identity"
  input: "save/RestoreV2:71"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_71"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/beta/adam_v"
  input: "save/Identity_72"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_73"
  op: "Identity"
  input: "save/RestoreV2:72"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_72"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma"
  input: "save/Identity_73"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_74"
  op: "Identity"
  input: "save/RestoreV2:73"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_73"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_74"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_75"
  op: "Identity"
  input: "save/RestoreV2:74"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_74"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_75"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_76"
  op: "Identity"
  input: "save/RestoreV2:75"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_75"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias"
  input: "save/Identity_76"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_77"
  op: "Identity"
  input: "save/RestoreV2:76"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_76"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_m"
  input: "save/Identity_77"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_78"
  op: "Identity"
  input: "save/RestoreV2:77"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_77"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/bias/adam_v"
  input: "save/Identity_78"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_79"
  op: "Identity"
  input: "save/RestoreV2:78"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_78"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel"
  input: "save/Identity_79"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_80"
  op: "Identity"
  input: "save/RestoreV2:79"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_79"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_m"
  input: "save/Identity_80"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_81"
  op: "Identity"
  input: "save/RestoreV2:80"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_80"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_0/output/dense/kernel/adam_v"
  input: "save/Identity_81"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_82"
  op: "Identity"
  input: "save/RestoreV2:81"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_81"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta"
  input: "save/Identity_82"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_83"
  op: "Identity"
  input: "save/RestoreV2:82"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_82"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_83"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_84"
  op: "Identity"
  input: "save/RestoreV2:83"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_83"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_84"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_85"
  op: "Identity"
  input: "save/RestoreV2:84"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_84"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma"
  input: "save/Identity_85"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_86"
  op: "Identity"
  input: "save/RestoreV2:85"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_85"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_86"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_87"
  op: "Identity"
  input: "save/RestoreV2:86"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_86"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_87"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_88"
  op: "Identity"
  input: "save/RestoreV2:87"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_87"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias"
  input: "save/Identity_88"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_89"
  op: "Identity"
  input: "save/RestoreV2:88"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_88"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_m"
  input: "save/Identity_89"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_90"
  op: "Identity"
  input: "save/RestoreV2:89"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_89"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/bias/adam_v"
  input: "save/Identity_90"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_91"
  op: "Identity"
  input: "save/RestoreV2:90"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_90"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel"
  input: "save/Identity_91"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_92"
  op: "Identity"
  input: "save/RestoreV2:91"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_91"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_m"
  input: "save/Identity_92"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_93"
  op: "Identity"
  input: "save/RestoreV2:92"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_92"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/output/dense/kernel/adam_v"
  input: "save/Identity_93"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_94"
  op: "Identity"
  input: "save/RestoreV2:93"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_93"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias"
  input: "save/Identity_94"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_95"
  op: "Identity"
  input: "save/RestoreV2:94"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_94"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_m"
  input: "save/Identity_95"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_96"
  op: "Identity"
  input: "save/RestoreV2:95"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_95"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/bias/adam_v"
  input: "save/Identity_96"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_97"
  op: "Identity"
  input: "save/RestoreV2:96"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_96"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel"
  input: "save/Identity_97"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_98"
  op: "Identity"
  input: "save/RestoreV2:97"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_97"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_m"
  input: "save/Identity_98"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_99"
  op: "Identity"
  input: "save/RestoreV2:98"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_98"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/key/kernel/adam_v"
  input: "save/Identity_99"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_100"
  op: "Identity"
  input: "save/RestoreV2:99"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_99"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias"
  input: "save/Identity_100"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_101"
  op: "Identity"
  input: "save/RestoreV2:100"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_100"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_m"
  input: "save/Identity_101"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_102"
  op: "Identity"
  input: "save/RestoreV2:101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_101"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/bias/adam_v"
  input: "save/Identity_102"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_103"
  op: "Identity"
  input: "save/RestoreV2:102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_102"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel"
  input: "save/Identity_103"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_104"
  op: "Identity"
  input: "save/RestoreV2:103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_103"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_m"
  input: "save/Identity_104"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_105"
  op: "Identity"
  input: "save/RestoreV2:104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_104"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/query/kernel/adam_v"
  input: "save/Identity_105"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_106"
  op: "Identity"
  input: "save/RestoreV2:105"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_105"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias"
  input: "save/Identity_106"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_107"
  op: "Identity"
  input: "save/RestoreV2:106"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_106"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_m"
  input: "save/Identity_107"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_108"
  op: "Identity"
  input: "save/RestoreV2:107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_107"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/bias/adam_v"
  input: "save/Identity_108"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_109"
  op: "Identity"
  input: "save/RestoreV2:108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_108"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel"
  input: "save/Identity_109"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_110"
  op: "Identity"
  input: "save/RestoreV2:109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_109"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_m"
  input: "save/Identity_110"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_111"
  op: "Identity"
  input: "save/RestoreV2:110"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_110"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/attention/self/value/kernel/adam_v"
  input: "save/Identity_111"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_112"
  op: "Identity"
  input: "save/RestoreV2:111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_111"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias"
  input: "save/Identity_112"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_113"
  op: "Identity"
  input: "save/RestoreV2:112"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_112"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_m"
  input: "save/Identity_113"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_114"
  op: "Identity"
  input: "save/RestoreV2:113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_113"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/bias/adam_v"
  input: "save/Identity_114"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_115"
  op: "Identity"
  input: "save/RestoreV2:114"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_114"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel"
  input: "save/Identity_115"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_116"
  op: "Identity"
  input: "save/RestoreV2:115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_115"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_m"
  input: "save/Identity_116"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_117"
  op: "Identity"
  input: "save/RestoreV2:116"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_116"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/intermediate/dense/kernel/adam_v"
  input: "save/Identity_117"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_118"
  op: "Identity"
  input: "save/RestoreV2:117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_117"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta"
  input: "save/Identity_118"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_119"
  op: "Identity"
  input: "save/RestoreV2:118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_118"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_m"
  input: "save/Identity_119"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_120"
  op: "Identity"
  input: "save/RestoreV2:119"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_119"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/beta/adam_v"
  input: "save/Identity_120"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_121"
  op: "Identity"
  input: "save/RestoreV2:120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_120"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma"
  input: "save/Identity_121"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_122"
  op: "Identity"
  input: "save/RestoreV2:121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_121"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_122"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_123"
  op: "Identity"
  input: "save/RestoreV2:122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_122"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_123"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_124"
  op: "Identity"
  input: "save/RestoreV2:123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_123"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias"
  input: "save/Identity_124"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_125"
  op: "Identity"
  input: "save/RestoreV2:124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_124"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_m"
  input: "save/Identity_125"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_126"
  op: "Identity"
  input: "save/RestoreV2:125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_125"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/bias/adam_v"
  input: "save/Identity_126"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_127"
  op: "Identity"
  input: "save/RestoreV2:126"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_126"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel"
  input: "save/Identity_127"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_128"
  op: "Identity"
  input: "save/RestoreV2:127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_127"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_m"
  input: "save/Identity_128"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_129"
  op: "Identity"
  input: "save/RestoreV2:128"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_128"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_1/output/dense/kernel/adam_v"
  input: "save/Identity_129"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_130"
  op: "Identity"
  input: "save/RestoreV2:129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_129"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta"
  input: "save/Identity_130"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_131"
  op: "Identity"
  input: "save/RestoreV2:130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_130"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_131"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_132"
  op: "Identity"
  input: "save/RestoreV2:131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_131"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_132"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_133"
  op: "Identity"
  input: "save/RestoreV2:132"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_132"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma"
  input: "save/Identity_133"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_134"
  op: "Identity"
  input: "save/RestoreV2:133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_133"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_134"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_135"
  op: "Identity"
  input: "save/RestoreV2:134"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_134"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_135"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_136"
  op: "Identity"
  input: "save/RestoreV2:135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_135"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias"
  input: "save/Identity_136"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_137"
  op: "Identity"
  input: "save/RestoreV2:136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_136"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_m"
  input: "save/Identity_137"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_138"
  op: "Identity"
  input: "save/RestoreV2:137"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_137"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/bias/adam_v"
  input: "save/Identity_138"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_139"
  op: "Identity"
  input: "save/RestoreV2:138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_138"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel"
  input: "save/Identity_139"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_140"
  op: "Identity"
  input: "save/RestoreV2:139"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_139"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_m"
  input: "save/Identity_140"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_141"
  op: "Identity"
  input: "save/RestoreV2:140"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_140"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/output/dense/kernel/adam_v"
  input: "save/Identity_141"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_142"
  op: "Identity"
  input: "save/RestoreV2:141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_141"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias"
  input: "save/Identity_142"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_143"
  op: "Identity"
  input: "save/RestoreV2:142"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_142"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_m"
  input: "save/Identity_143"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_144"
  op: "Identity"
  input: "save/RestoreV2:143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_143"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/bias/adam_v"
  input: "save/Identity_144"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_145"
  op: "Identity"
  input: "save/RestoreV2:144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_144"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel"
  input: "save/Identity_145"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_146"
  op: "Identity"
  input: "save/RestoreV2:145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_145"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_m"
  input: "save/Identity_146"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_147"
  op: "Identity"
  input: "save/RestoreV2:146"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_146"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/key/kernel/adam_v"
  input: "save/Identity_147"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_148"
  op: "Identity"
  input: "save/RestoreV2:147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_147"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias"
  input: "save/Identity_148"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_149"
  op: "Identity"
  input: "save/RestoreV2:148"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_148"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_m"
  input: "save/Identity_149"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_150"
  op: "Identity"
  input: "save/RestoreV2:149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_149"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/bias/adam_v"
  input: "save/Identity_150"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_151"
  op: "Identity"
  input: "save/RestoreV2:150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_150"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel"
  input: "save/Identity_151"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_152"
  op: "Identity"
  input: "save/RestoreV2:151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_151"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_m"
  input: "save/Identity_152"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_153"
  op: "Identity"
  input: "save/RestoreV2:152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_152"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/query/kernel/adam_v"
  input: "save/Identity_153"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_154"
  op: "Identity"
  input: "save/RestoreV2:153"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_153"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias"
  input: "save/Identity_154"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_155"
  op: "Identity"
  input: "save/RestoreV2:154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_154"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_m"
  input: "save/Identity_155"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_156"
  op: "Identity"
  input: "save/RestoreV2:155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_155"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/bias/adam_v"
  input: "save/Identity_156"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_157"
  op: "Identity"
  input: "save/RestoreV2:156"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_156"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel"
  input: "save/Identity_157"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_158"
  op: "Identity"
  input: "save/RestoreV2:157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_157"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_m"
  input: "save/Identity_158"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_159"
  op: "Identity"
  input: "save/RestoreV2:158"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_158"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/attention/self/value/kernel/adam_v"
  input: "save/Identity_159"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_160"
  op: "Identity"
  input: "save/RestoreV2:159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_159"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias"
  input: "save/Identity_160"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_161"
  op: "Identity"
  input: "save/RestoreV2:160"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_160"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_m"
  input: "save/Identity_161"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_162"
  op: "Identity"
  input: "save/RestoreV2:161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_161"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/bias/adam_v"
  input: "save/Identity_162"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_163"
  op: "Identity"
  input: "save/RestoreV2:162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_162"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel"
  input: "save/Identity_163"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_164"
  op: "Identity"
  input: "save/RestoreV2:163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_163"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_m"
  input: "save/Identity_164"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_165"
  op: "Identity"
  input: "save/RestoreV2:164"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_164"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/intermediate/dense/kernel/adam_v"
  input: "save/Identity_165"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_166"
  op: "Identity"
  input: "save/RestoreV2:165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_165"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta"
  input: "save/Identity_166"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_167"
  op: "Identity"
  input: "save/RestoreV2:166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_166"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_m"
  input: "save/Identity_167"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_168"
  op: "Identity"
  input: "save/RestoreV2:167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_167"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/beta/adam_v"
  input: "save/Identity_168"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_169"
  op: "Identity"
  input: "save/RestoreV2:168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_168"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma"
  input: "save/Identity_169"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_170"
  op: "Identity"
  input: "save/RestoreV2:169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_169"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_170"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_171"
  op: "Identity"
  input: "save/RestoreV2:170"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_170"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_171"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_172"
  op: "Identity"
  input: "save/RestoreV2:171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_171"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias"
  input: "save/Identity_172"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_173"
  op: "Identity"
  input: "save/RestoreV2:172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_172"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_m"
  input: "save/Identity_173"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_174"
  op: "Identity"
  input: "save/RestoreV2:173"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_173"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/bias/adam_v"
  input: "save/Identity_174"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_175"
  op: "Identity"
  input: "save/RestoreV2:174"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_174"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel"
  input: "save/Identity_175"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_176"
  op: "Identity"
  input: "save/RestoreV2:175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_175"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_m"
  input: "save/Identity_176"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_177"
  op: "Identity"
  input: "save/RestoreV2:176"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_176"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_10/output/dense/kernel/adam_v"
  input: "save/Identity_177"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_178"
  op: "Identity"
  input: "save/RestoreV2:177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_177"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta"
  input: "save/Identity_178"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_179"
  op: "Identity"
  input: "save/RestoreV2:178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_178"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_179"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_180"
  op: "Identity"
  input: "save/RestoreV2:179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_179"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_180"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_181"
  op: "Identity"
  input: "save/RestoreV2:180"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_180"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma"
  input: "save/Identity_181"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_182"
  op: "Identity"
  input: "save/RestoreV2:181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_181"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_182"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_183"
  op: "Identity"
  input: "save/RestoreV2:182"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_182"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_183"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_184"
  op: "Identity"
  input: "save/RestoreV2:183"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_183"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias"
  input: "save/Identity_184"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_185"
  op: "Identity"
  input: "save/RestoreV2:184"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_184"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_m"
  input: "save/Identity_185"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_186"
  op: "Identity"
  input: "save/RestoreV2:185"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_185"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/bias/adam_v"
  input: "save/Identity_186"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_187"
  op: "Identity"
  input: "save/RestoreV2:186"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_186"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel"
  input: "save/Identity_187"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_188"
  op: "Identity"
  input: "save/RestoreV2:187"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_187"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_m"
  input: "save/Identity_188"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_189"
  op: "Identity"
  input: "save/RestoreV2:188"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_188"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/output/dense/kernel/adam_v"
  input: "save/Identity_189"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_190"
  op: "Identity"
  input: "save/RestoreV2:189"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_189"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias"
  input: "save/Identity_190"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_191"
  op: "Identity"
  input: "save/RestoreV2:190"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_190"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_m"
  input: "save/Identity_191"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_192"
  op: "Identity"
  input: "save/RestoreV2:191"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_191"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/bias/adam_v"
  input: "save/Identity_192"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_193"
  op: "Identity"
  input: "save/RestoreV2:192"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_192"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel"
  input: "save/Identity_193"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_194"
  op: "Identity"
  input: "save/RestoreV2:193"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_193"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_m"
  input: "save/Identity_194"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_195"
  op: "Identity"
  input: "save/RestoreV2:194"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_194"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/key/kernel/adam_v"
  input: "save/Identity_195"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_196"
  op: "Identity"
  input: "save/RestoreV2:195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_195"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias"
  input: "save/Identity_196"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_197"
  op: "Identity"
  input: "save/RestoreV2:196"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_196"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_m"
  input: "save/Identity_197"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_198"
  op: "Identity"
  input: "save/RestoreV2:197"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_197"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/bias/adam_v"
  input: "save/Identity_198"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_199"
  op: "Identity"
  input: "save/RestoreV2:198"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_198"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel"
  input: "save/Identity_199"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_200"
  op: "Identity"
  input: "save/RestoreV2:199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_199"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_m"
  input: "save/Identity_200"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_201"
  op: "Identity"
  input: "save/RestoreV2:200"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_200"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/query/kernel/adam_v"
  input: "save/Identity_201"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_202"
  op: "Identity"
  input: "save/RestoreV2:201"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_201"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias"
  input: "save/Identity_202"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_203"
  op: "Identity"
  input: "save/RestoreV2:202"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_202"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_m"
  input: "save/Identity_203"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_204"
  op: "Identity"
  input: "save/RestoreV2:203"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_203"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/bias/adam_v"
  input: "save/Identity_204"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_205"
  op: "Identity"
  input: "save/RestoreV2:204"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_204"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel"
  input: "save/Identity_205"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_206"
  op: "Identity"
  input: "save/RestoreV2:205"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_205"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_m"
  input: "save/Identity_206"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_207"
  op: "Identity"
  input: "save/RestoreV2:206"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_206"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/attention/self/value/kernel/adam_v"
  input: "save/Identity_207"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_208"
  op: "Identity"
  input: "save/RestoreV2:207"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_207"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias"
  input: "save/Identity_208"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_209"
  op: "Identity"
  input: "save/RestoreV2:208"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_208"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_m"
  input: "save/Identity_209"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_210"
  op: "Identity"
  input: "save/RestoreV2:209"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_209"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/bias/adam_v"
  input: "save/Identity_210"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_211"
  op: "Identity"
  input: "save/RestoreV2:210"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_210"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel"
  input: "save/Identity_211"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_212"
  op: "Identity"
  input: "save/RestoreV2:211"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_211"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_m"
  input: "save/Identity_212"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_213"
  op: "Identity"
  input: "save/RestoreV2:212"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_212"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/intermediate/dense/kernel/adam_v"
  input: "save/Identity_213"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_214"
  op: "Identity"
  input: "save/RestoreV2:213"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_213"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta"
  input: "save/Identity_214"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_215"
  op: "Identity"
  input: "save/RestoreV2:214"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_214"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_m"
  input: "save/Identity_215"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_216"
  op: "Identity"
  input: "save/RestoreV2:215"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_215"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/beta/adam_v"
  input: "save/Identity_216"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_217"
  op: "Identity"
  input: "save/RestoreV2:216"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_216"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma"
  input: "save/Identity_217"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_218"
  op: "Identity"
  input: "save/RestoreV2:217"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_217"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_218"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_219"
  op: "Identity"
  input: "save/RestoreV2:218"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_218"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_219"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_220"
  op: "Identity"
  input: "save/RestoreV2:219"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_219"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias"
  input: "save/Identity_220"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_221"
  op: "Identity"
  input: "save/RestoreV2:220"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_220"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_m"
  input: "save/Identity_221"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_222"
  op: "Identity"
  input: "save/RestoreV2:221"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_221"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/bias/adam_v"
  input: "save/Identity_222"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_223"
  op: "Identity"
  input: "save/RestoreV2:222"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_222"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel"
  input: "save/Identity_223"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_224"
  op: "Identity"
  input: "save/RestoreV2:223"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_223"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_m"
  input: "save/Identity_224"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_225"
  op: "Identity"
  input: "save/RestoreV2:224"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_224"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_11/output/dense/kernel/adam_v"
  input: "save/Identity_225"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_226"
  op: "Identity"
  input: "save/RestoreV2:225"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_225"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta"
  input: "save/Identity_226"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_227"
  op: "Identity"
  input: "save/RestoreV2:226"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_226"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_227"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_228"
  op: "Identity"
  input: "save/RestoreV2:227"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_227"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_228"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_229"
  op: "Identity"
  input: "save/RestoreV2:228"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_228"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma"
  input: "save/Identity_229"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_230"
  op: "Identity"
  input: "save/RestoreV2:229"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_229"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_230"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_231"
  op: "Identity"
  input: "save/RestoreV2:230"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_230"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_231"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_232"
  op: "Identity"
  input: "save/RestoreV2:231"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_231"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias"
  input: "save/Identity_232"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_233"
  op: "Identity"
  input: "save/RestoreV2:232"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_232"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_m"
  input: "save/Identity_233"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_234"
  op: "Identity"
  input: "save/RestoreV2:233"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_233"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/bias/adam_v"
  input: "save/Identity_234"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_235"
  op: "Identity"
  input: "save/RestoreV2:234"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_234"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel"
  input: "save/Identity_235"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_236"
  op: "Identity"
  input: "save/RestoreV2:235"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_235"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_m"
  input: "save/Identity_236"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_237"
  op: "Identity"
  input: "save/RestoreV2:236"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_236"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/output/dense/kernel/adam_v"
  input: "save/Identity_237"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_238"
  op: "Identity"
  input: "save/RestoreV2:237"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_237"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias"
  input: "save/Identity_238"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_239"
  op: "Identity"
  input: "save/RestoreV2:238"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_238"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_m"
  input: "save/Identity_239"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_240"
  op: "Identity"
  input: "save/RestoreV2:239"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_239"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/bias/adam_v"
  input: "save/Identity_240"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_241"
  op: "Identity"
  input: "save/RestoreV2:240"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_240"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel"
  input: "save/Identity_241"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_242"
  op: "Identity"
  input: "save/RestoreV2:241"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_241"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_m"
  input: "save/Identity_242"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_243"
  op: "Identity"
  input: "save/RestoreV2:242"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_242"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/key/kernel/adam_v"
  input: "save/Identity_243"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_244"
  op: "Identity"
  input: "save/RestoreV2:243"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_243"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias"
  input: "save/Identity_244"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_245"
  op: "Identity"
  input: "save/RestoreV2:244"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_244"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_m"
  input: "save/Identity_245"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_246"
  op: "Identity"
  input: "save/RestoreV2:245"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_245"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/bias/adam_v"
  input: "save/Identity_246"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_247"
  op: "Identity"
  input: "save/RestoreV2:246"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_246"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel"
  input: "save/Identity_247"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_248"
  op: "Identity"
  input: "save/RestoreV2:247"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_247"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_m"
  input: "save/Identity_248"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_249"
  op: "Identity"
  input: "save/RestoreV2:248"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_248"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/query/kernel/adam_v"
  input: "save/Identity_249"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_250"
  op: "Identity"
  input: "save/RestoreV2:249"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_249"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias"
  input: "save/Identity_250"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_251"
  op: "Identity"
  input: "save/RestoreV2:250"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_250"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_m"
  input: "save/Identity_251"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_252"
  op: "Identity"
  input: "save/RestoreV2:251"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_251"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/bias/adam_v"
  input: "save/Identity_252"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_253"
  op: "Identity"
  input: "save/RestoreV2:252"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_252"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel"
  input: "save/Identity_253"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_254"
  op: "Identity"
  input: "save/RestoreV2:253"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_253"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_m"
  input: "save/Identity_254"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_255"
  op: "Identity"
  input: "save/RestoreV2:254"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_254"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/attention/self/value/kernel/adam_v"
  input: "save/Identity_255"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_256"
  op: "Identity"
  input: "save/RestoreV2:255"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_255"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias"
  input: "save/Identity_256"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_257"
  op: "Identity"
  input: "save/RestoreV2:256"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_256"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_m"
  input: "save/Identity_257"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_258"
  op: "Identity"
  input: "save/RestoreV2:257"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_257"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/bias/adam_v"
  input: "save/Identity_258"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_259"
  op: "Identity"
  input: "save/RestoreV2:258"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_258"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel"
  input: "save/Identity_259"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_260"
  op: "Identity"
  input: "save/RestoreV2:259"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_259"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_m"
  input: "save/Identity_260"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_261"
  op: "Identity"
  input: "save/RestoreV2:260"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_260"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/intermediate/dense/kernel/adam_v"
  input: "save/Identity_261"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_262"
  op: "Identity"
  input: "save/RestoreV2:261"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_261"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta"
  input: "save/Identity_262"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_263"
  op: "Identity"
  input: "save/RestoreV2:262"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_262"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_m"
  input: "save/Identity_263"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_264"
  op: "Identity"
  input: "save/RestoreV2:263"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_263"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/beta/adam_v"
  input: "save/Identity_264"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_265"
  op: "Identity"
  input: "save/RestoreV2:264"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_264"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma"
  input: "save/Identity_265"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_266"
  op: "Identity"
  input: "save/RestoreV2:265"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_265"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_266"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_267"
  op: "Identity"
  input: "save/RestoreV2:266"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_266"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_267"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_268"
  op: "Identity"
  input: "save/RestoreV2:267"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_267"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias"
  input: "save/Identity_268"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_269"
  op: "Identity"
  input: "save/RestoreV2:268"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_268"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_m"
  input: "save/Identity_269"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_270"
  op: "Identity"
  input: "save/RestoreV2:269"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_269"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/bias/adam_v"
  input: "save/Identity_270"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_271"
  op: "Identity"
  input: "save/RestoreV2:270"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_270"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel"
  input: "save/Identity_271"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_272"
  op: "Identity"
  input: "save/RestoreV2:271"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_271"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_m"
  input: "save/Identity_272"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_273"
  op: "Identity"
  input: "save/RestoreV2:272"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_272"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_2/output/dense/kernel/adam_v"
  input: "save/Identity_273"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_274"
  op: "Identity"
  input: "save/RestoreV2:273"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_273"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta"
  input: "save/Identity_274"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_275"
  op: "Identity"
  input: "save/RestoreV2:274"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_274"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_275"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_276"
  op: "Identity"
  input: "save/RestoreV2:275"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_275"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_276"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_277"
  op: "Identity"
  input: "save/RestoreV2:276"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_276"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma"
  input: "save/Identity_277"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_278"
  op: "Identity"
  input: "save/RestoreV2:277"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_277"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_278"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_279"
  op: "Identity"
  input: "save/RestoreV2:278"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_278"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_279"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_280"
  op: "Identity"
  input: "save/RestoreV2:279"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_279"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias"
  input: "save/Identity_280"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_281"
  op: "Identity"
  input: "save/RestoreV2:280"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_280"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_m"
  input: "save/Identity_281"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_282"
  op: "Identity"
  input: "save/RestoreV2:281"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_281"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/bias/adam_v"
  input: "save/Identity_282"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_283"
  op: "Identity"
  input: "save/RestoreV2:282"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_282"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel"
  input: "save/Identity_283"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_284"
  op: "Identity"
  input: "save/RestoreV2:283"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_283"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_m"
  input: "save/Identity_284"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_285"
  op: "Identity"
  input: "save/RestoreV2:284"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_284"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/output/dense/kernel/adam_v"
  input: "save/Identity_285"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_286"
  op: "Identity"
  input: "save/RestoreV2:285"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_285"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias"
  input: "save/Identity_286"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_287"
  op: "Identity"
  input: "save/RestoreV2:286"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_286"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_m"
  input: "save/Identity_287"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_288"
  op: "Identity"
  input: "save/RestoreV2:287"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_287"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/bias/adam_v"
  input: "save/Identity_288"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_289"
  op: "Identity"
  input: "save/RestoreV2:288"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_288"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel"
  input: "save/Identity_289"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_290"
  op: "Identity"
  input: "save/RestoreV2:289"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_289"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_m"
  input: "save/Identity_290"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_291"
  op: "Identity"
  input: "save/RestoreV2:290"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_290"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/key/kernel/adam_v"
  input: "save/Identity_291"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_292"
  op: "Identity"
  input: "save/RestoreV2:291"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_291"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias"
  input: "save/Identity_292"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_293"
  op: "Identity"
  input: "save/RestoreV2:292"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_292"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_m"
  input: "save/Identity_293"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_294"
  op: "Identity"
  input: "save/RestoreV2:293"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_293"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/bias/adam_v"
  input: "save/Identity_294"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_295"
  op: "Identity"
  input: "save/RestoreV2:294"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_294"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel"
  input: "save/Identity_295"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_296"
  op: "Identity"
  input: "save/RestoreV2:295"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_295"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_m"
  input: "save/Identity_296"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_297"
  op: "Identity"
  input: "save/RestoreV2:296"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_296"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/query/kernel/adam_v"
  input: "save/Identity_297"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_298"
  op: "Identity"
  input: "save/RestoreV2:297"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_297"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias"
  input: "save/Identity_298"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_299"
  op: "Identity"
  input: "save/RestoreV2:298"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_298"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_m"
  input: "save/Identity_299"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_300"
  op: "Identity"
  input: "save/RestoreV2:299"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_299"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/bias/adam_v"
  input: "save/Identity_300"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_301"
  op: "Identity"
  input: "save/RestoreV2:300"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_300"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel"
  input: "save/Identity_301"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_302"
  op: "Identity"
  input: "save/RestoreV2:301"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_301"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_m"
  input: "save/Identity_302"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_303"
  op: "Identity"
  input: "save/RestoreV2:302"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_302"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/attention/self/value/kernel/adam_v"
  input: "save/Identity_303"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_304"
  op: "Identity"
  input: "save/RestoreV2:303"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_303"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias"
  input: "save/Identity_304"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_305"
  op: "Identity"
  input: "save/RestoreV2:304"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_304"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_m"
  input: "save/Identity_305"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_306"
  op: "Identity"
  input: "save/RestoreV2:305"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_305"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/bias/adam_v"
  input: "save/Identity_306"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_307"
  op: "Identity"
  input: "save/RestoreV2:306"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_306"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel"
  input: "save/Identity_307"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_308"
  op: "Identity"
  input: "save/RestoreV2:307"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_307"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_m"
  input: "save/Identity_308"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_309"
  op: "Identity"
  input: "save/RestoreV2:308"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_308"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/intermediate/dense/kernel/adam_v"
  input: "save/Identity_309"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_310"
  op: "Identity"
  input: "save/RestoreV2:309"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_309"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta"
  input: "save/Identity_310"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_311"
  op: "Identity"
  input: "save/RestoreV2:310"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_310"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_m"
  input: "save/Identity_311"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_312"
  op: "Identity"
  input: "save/RestoreV2:311"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_311"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/beta/adam_v"
  input: "save/Identity_312"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_313"
  op: "Identity"
  input: "save/RestoreV2:312"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_312"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma"
  input: "save/Identity_313"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_314"
  op: "Identity"
  input: "save/RestoreV2:313"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_313"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_314"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_315"
  op: "Identity"
  input: "save/RestoreV2:314"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_314"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_315"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_316"
  op: "Identity"
  input: "save/RestoreV2:315"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_315"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias"
  input: "save/Identity_316"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_317"
  op: "Identity"
  input: "save/RestoreV2:316"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_316"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_m"
  input: "save/Identity_317"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_318"
  op: "Identity"
  input: "save/RestoreV2:317"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_317"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/bias/adam_v"
  input: "save/Identity_318"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_319"
  op: "Identity"
  input: "save/RestoreV2:318"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_318"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel"
  input: "save/Identity_319"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_320"
  op: "Identity"
  input: "save/RestoreV2:319"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_319"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_m"
  input: "save/Identity_320"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_321"
  op: "Identity"
  input: "save/RestoreV2:320"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_320"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_3/output/dense/kernel/adam_v"
  input: "save/Identity_321"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_322"
  op: "Identity"
  input: "save/RestoreV2:321"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_321"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta"
  input: "save/Identity_322"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_323"
  op: "Identity"
  input: "save/RestoreV2:322"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_322"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_323"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_324"
  op: "Identity"
  input: "save/RestoreV2:323"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_323"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_324"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_325"
  op: "Identity"
  input: "save/RestoreV2:324"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_324"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma"
  input: "save/Identity_325"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_326"
  op: "Identity"
  input: "save/RestoreV2:325"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_325"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_326"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_327"
  op: "Identity"
  input: "save/RestoreV2:326"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_326"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_327"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_328"
  op: "Identity"
  input: "save/RestoreV2:327"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_327"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias"
  input: "save/Identity_328"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_329"
  op: "Identity"
  input: "save/RestoreV2:328"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_328"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_m"
  input: "save/Identity_329"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_330"
  op: "Identity"
  input: "save/RestoreV2:329"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_329"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/bias/adam_v"
  input: "save/Identity_330"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_331"
  op: "Identity"
  input: "save/RestoreV2:330"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_330"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel"
  input: "save/Identity_331"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_332"
  op: "Identity"
  input: "save/RestoreV2:331"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_331"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_m"
  input: "save/Identity_332"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_333"
  op: "Identity"
  input: "save/RestoreV2:332"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_332"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/output/dense/kernel/adam_v"
  input: "save/Identity_333"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_334"
  op: "Identity"
  input: "save/RestoreV2:333"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_333"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias"
  input: "save/Identity_334"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_335"
  op: "Identity"
  input: "save/RestoreV2:334"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_334"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_m"
  input: "save/Identity_335"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_336"
  op: "Identity"
  input: "save/RestoreV2:335"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_335"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/bias/adam_v"
  input: "save/Identity_336"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_337"
  op: "Identity"
  input: "save/RestoreV2:336"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_336"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel"
  input: "save/Identity_337"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_338"
  op: "Identity"
  input: "save/RestoreV2:337"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_337"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_m"
  input: "save/Identity_338"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_339"
  op: "Identity"
  input: "save/RestoreV2:338"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_338"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/key/kernel/adam_v"
  input: "save/Identity_339"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_340"
  op: "Identity"
  input: "save/RestoreV2:339"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_339"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias"
  input: "save/Identity_340"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_341"
  op: "Identity"
  input: "save/RestoreV2:340"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_340"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_m"
  input: "save/Identity_341"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_342"
  op: "Identity"
  input: "save/RestoreV2:341"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_341"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/bias/adam_v"
  input: "save/Identity_342"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_343"
  op: "Identity"
  input: "save/RestoreV2:342"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_342"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel"
  input: "save/Identity_343"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_344"
  op: "Identity"
  input: "save/RestoreV2:343"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_343"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_m"
  input: "save/Identity_344"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_345"
  op: "Identity"
  input: "save/RestoreV2:344"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_344"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/query/kernel/adam_v"
  input: "save/Identity_345"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_346"
  op: "Identity"
  input: "save/RestoreV2:345"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_345"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias"
  input: "save/Identity_346"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_347"
  op: "Identity"
  input: "save/RestoreV2:346"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_346"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_m"
  input: "save/Identity_347"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_348"
  op: "Identity"
  input: "save/RestoreV2:347"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_347"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/bias/adam_v"
  input: "save/Identity_348"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_349"
  op: "Identity"
  input: "save/RestoreV2:348"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_348"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel"
  input: "save/Identity_349"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_350"
  op: "Identity"
  input: "save/RestoreV2:349"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_349"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_m"
  input: "save/Identity_350"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_351"
  op: "Identity"
  input: "save/RestoreV2:350"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_350"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/attention/self/value/kernel/adam_v"
  input: "save/Identity_351"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_352"
  op: "Identity"
  input: "save/RestoreV2:351"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_351"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias"
  input: "save/Identity_352"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_353"
  op: "Identity"
  input: "save/RestoreV2:352"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_352"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_m"
  input: "save/Identity_353"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_354"
  op: "Identity"
  input: "save/RestoreV2:353"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_353"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/bias/adam_v"
  input: "save/Identity_354"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_355"
  op: "Identity"
  input: "save/RestoreV2:354"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_354"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel"
  input: "save/Identity_355"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_356"
  op: "Identity"
  input: "save/RestoreV2:355"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_355"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_m"
  input: "save/Identity_356"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_357"
  op: "Identity"
  input: "save/RestoreV2:356"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_356"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/intermediate/dense/kernel/adam_v"
  input: "save/Identity_357"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_358"
  op: "Identity"
  input: "save/RestoreV2:357"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_357"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta"
  input: "save/Identity_358"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_359"
  op: "Identity"
  input: "save/RestoreV2:358"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_358"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_m"
  input: "save/Identity_359"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_360"
  op: "Identity"
  input: "save/RestoreV2:359"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_359"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/beta/adam_v"
  input: "save/Identity_360"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_361"
  op: "Identity"
  input: "save/RestoreV2:360"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_360"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma"
  input: "save/Identity_361"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_362"
  op: "Identity"
  input: "save/RestoreV2:361"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_361"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_362"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_363"
  op: "Identity"
  input: "save/RestoreV2:362"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_362"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_363"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_364"
  op: "Identity"
  input: "save/RestoreV2:363"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_363"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias"
  input: "save/Identity_364"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_365"
  op: "Identity"
  input: "save/RestoreV2:364"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_364"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_m"
  input: "save/Identity_365"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_366"
  op: "Identity"
  input: "save/RestoreV2:365"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_365"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/bias/adam_v"
  input: "save/Identity_366"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_367"
  op: "Identity"
  input: "save/RestoreV2:366"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_366"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel"
  input: "save/Identity_367"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_368"
  op: "Identity"
  input: "save/RestoreV2:367"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_367"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_m"
  input: "save/Identity_368"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_369"
  op: "Identity"
  input: "save/RestoreV2:368"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_368"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_4/output/dense/kernel/adam_v"
  input: "save/Identity_369"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_370"
  op: "Identity"
  input: "save/RestoreV2:369"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_369"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta"
  input: "save/Identity_370"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_371"
  op: "Identity"
  input: "save/RestoreV2:370"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_370"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_371"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_372"
  op: "Identity"
  input: "save/RestoreV2:371"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_371"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_372"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_373"
  op: "Identity"
  input: "save/RestoreV2:372"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_372"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma"
  input: "save/Identity_373"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_374"
  op: "Identity"
  input: "save/RestoreV2:373"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_373"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_374"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_375"
  op: "Identity"
  input: "save/RestoreV2:374"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_374"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_375"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_376"
  op: "Identity"
  input: "save/RestoreV2:375"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_375"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias"
  input: "save/Identity_376"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_377"
  op: "Identity"
  input: "save/RestoreV2:376"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_376"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_m"
  input: "save/Identity_377"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_378"
  op: "Identity"
  input: "save/RestoreV2:377"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_377"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/bias/adam_v"
  input: "save/Identity_378"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_379"
  op: "Identity"
  input: "save/RestoreV2:378"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_378"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel"
  input: "save/Identity_379"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_380"
  op: "Identity"
  input: "save/RestoreV2:379"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_379"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_m"
  input: "save/Identity_380"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_381"
  op: "Identity"
  input: "save/RestoreV2:380"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_380"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/output/dense/kernel/adam_v"
  input: "save/Identity_381"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_382"
  op: "Identity"
  input: "save/RestoreV2:381"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_381"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias"
  input: "save/Identity_382"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_383"
  op: "Identity"
  input: "save/RestoreV2:382"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_382"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_m"
  input: "save/Identity_383"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_384"
  op: "Identity"
  input: "save/RestoreV2:383"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_383"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/bias/adam_v"
  input: "save/Identity_384"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_385"
  op: "Identity"
  input: "save/RestoreV2:384"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_384"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel"
  input: "save/Identity_385"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_386"
  op: "Identity"
  input: "save/RestoreV2:385"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_385"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_m"
  input: "save/Identity_386"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_387"
  op: "Identity"
  input: "save/RestoreV2:386"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_386"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/key/kernel/adam_v"
  input: "save/Identity_387"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_388"
  op: "Identity"
  input: "save/RestoreV2:387"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_387"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias"
  input: "save/Identity_388"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_389"
  op: "Identity"
  input: "save/RestoreV2:388"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_388"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_m"
  input: "save/Identity_389"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_390"
  op: "Identity"
  input: "save/RestoreV2:389"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_389"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/bias/adam_v"
  input: "save/Identity_390"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_391"
  op: "Identity"
  input: "save/RestoreV2:390"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_390"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel"
  input: "save/Identity_391"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_392"
  op: "Identity"
  input: "save/RestoreV2:391"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_391"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_m"
  input: "save/Identity_392"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_393"
  op: "Identity"
  input: "save/RestoreV2:392"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_392"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/query/kernel/adam_v"
  input: "save/Identity_393"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_394"
  op: "Identity"
  input: "save/RestoreV2:393"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_393"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias"
  input: "save/Identity_394"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_395"
  op: "Identity"
  input: "save/RestoreV2:394"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_394"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_m"
  input: "save/Identity_395"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_396"
  op: "Identity"
  input: "save/RestoreV2:395"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_395"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/bias/adam_v"
  input: "save/Identity_396"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_397"
  op: "Identity"
  input: "save/RestoreV2:396"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_396"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel"
  input: "save/Identity_397"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_398"
  op: "Identity"
  input: "save/RestoreV2:397"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_397"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_m"
  input: "save/Identity_398"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_399"
  op: "Identity"
  input: "save/RestoreV2:398"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_398"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/attention/self/value/kernel/adam_v"
  input: "save/Identity_399"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_400"
  op: "Identity"
  input: "save/RestoreV2:399"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_399"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias"
  input: "save/Identity_400"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_401"
  op: "Identity"
  input: "save/RestoreV2:400"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_400"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_m"
  input: "save/Identity_401"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_402"
  op: "Identity"
  input: "save/RestoreV2:401"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_401"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/bias/adam_v"
  input: "save/Identity_402"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_403"
  op: "Identity"
  input: "save/RestoreV2:402"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_402"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel"
  input: "save/Identity_403"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_404"
  op: "Identity"
  input: "save/RestoreV2:403"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_403"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_m"
  input: "save/Identity_404"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_405"
  op: "Identity"
  input: "save/RestoreV2:404"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_404"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/intermediate/dense/kernel/adam_v"
  input: "save/Identity_405"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_406"
  op: "Identity"
  input: "save/RestoreV2:405"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_405"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta"
  input: "save/Identity_406"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_407"
  op: "Identity"
  input: "save/RestoreV2:406"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_406"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_m"
  input: "save/Identity_407"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_408"
  op: "Identity"
  input: "save/RestoreV2:407"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_407"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/beta/adam_v"
  input: "save/Identity_408"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_409"
  op: "Identity"
  input: "save/RestoreV2:408"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_408"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma"
  input: "save/Identity_409"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_410"
  op: "Identity"
  input: "save/RestoreV2:409"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_409"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_410"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_411"
  op: "Identity"
  input: "save/RestoreV2:410"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_410"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_411"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_412"
  op: "Identity"
  input: "save/RestoreV2:411"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_411"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias"
  input: "save/Identity_412"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_413"
  op: "Identity"
  input: "save/RestoreV2:412"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_412"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_m"
  input: "save/Identity_413"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_414"
  op: "Identity"
  input: "save/RestoreV2:413"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_413"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/bias/adam_v"
  input: "save/Identity_414"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_415"
  op: "Identity"
  input: "save/RestoreV2:414"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_414"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel"
  input: "save/Identity_415"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_416"
  op: "Identity"
  input: "save/RestoreV2:415"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_415"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_m"
  input: "save/Identity_416"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_417"
  op: "Identity"
  input: "save/RestoreV2:416"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_416"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_5/output/dense/kernel/adam_v"
  input: "save/Identity_417"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_418"
  op: "Identity"
  input: "save/RestoreV2:417"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_417"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta"
  input: "save/Identity_418"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_419"
  op: "Identity"
  input: "save/RestoreV2:418"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_418"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_419"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_420"
  op: "Identity"
  input: "save/RestoreV2:419"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_419"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_420"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_421"
  op: "Identity"
  input: "save/RestoreV2:420"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_420"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma"
  input: "save/Identity_421"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_422"
  op: "Identity"
  input: "save/RestoreV2:421"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_421"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_422"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_423"
  op: "Identity"
  input: "save/RestoreV2:422"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_422"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_423"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_424"
  op: "Identity"
  input: "save/RestoreV2:423"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_423"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias"
  input: "save/Identity_424"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_425"
  op: "Identity"
  input: "save/RestoreV2:424"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_424"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_m"
  input: "save/Identity_425"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_426"
  op: "Identity"
  input: "save/RestoreV2:425"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_425"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/bias/adam_v"
  input: "save/Identity_426"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_427"
  op: "Identity"
  input: "save/RestoreV2:426"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_426"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel"
  input: "save/Identity_427"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_428"
  op: "Identity"
  input: "save/RestoreV2:427"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_427"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_m"
  input: "save/Identity_428"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_429"
  op: "Identity"
  input: "save/RestoreV2:428"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_428"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/output/dense/kernel/adam_v"
  input: "save/Identity_429"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_430"
  op: "Identity"
  input: "save/RestoreV2:429"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_429"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias"
  input: "save/Identity_430"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_431"
  op: "Identity"
  input: "save/RestoreV2:430"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_430"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_m"
  input: "save/Identity_431"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_432"
  op: "Identity"
  input: "save/RestoreV2:431"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_431"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/bias/adam_v"
  input: "save/Identity_432"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_433"
  op: "Identity"
  input: "save/RestoreV2:432"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_432"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel"
  input: "save/Identity_433"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_434"
  op: "Identity"
  input: "save/RestoreV2:433"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_433"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_m"
  input: "save/Identity_434"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_435"
  op: "Identity"
  input: "save/RestoreV2:434"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_434"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/key/kernel/adam_v"
  input: "save/Identity_435"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_436"
  op: "Identity"
  input: "save/RestoreV2:435"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_435"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias"
  input: "save/Identity_436"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_437"
  op: "Identity"
  input: "save/RestoreV2:436"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_436"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_m"
  input: "save/Identity_437"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_438"
  op: "Identity"
  input: "save/RestoreV2:437"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_437"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/bias/adam_v"
  input: "save/Identity_438"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_439"
  op: "Identity"
  input: "save/RestoreV2:438"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_438"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel"
  input: "save/Identity_439"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_440"
  op: "Identity"
  input: "save/RestoreV2:439"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_439"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_m"
  input: "save/Identity_440"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_441"
  op: "Identity"
  input: "save/RestoreV2:440"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_440"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/query/kernel/adam_v"
  input: "save/Identity_441"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_442"
  op: "Identity"
  input: "save/RestoreV2:441"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_441"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias"
  input: "save/Identity_442"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_443"
  op: "Identity"
  input: "save/RestoreV2:442"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_442"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_m"
  input: "save/Identity_443"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_444"
  op: "Identity"
  input: "save/RestoreV2:443"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_443"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/bias/adam_v"
  input: "save/Identity_444"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_445"
  op: "Identity"
  input: "save/RestoreV2:444"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_444"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel"
  input: "save/Identity_445"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_446"
  op: "Identity"
  input: "save/RestoreV2:445"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_445"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_m"
  input: "save/Identity_446"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_447"
  op: "Identity"
  input: "save/RestoreV2:446"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_446"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/attention/self/value/kernel/adam_v"
  input: "save/Identity_447"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_448"
  op: "Identity"
  input: "save/RestoreV2:447"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_447"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias"
  input: "save/Identity_448"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_449"
  op: "Identity"
  input: "save/RestoreV2:448"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_448"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_m"
  input: "save/Identity_449"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_450"
  op: "Identity"
  input: "save/RestoreV2:449"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_449"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/bias/adam_v"
  input: "save/Identity_450"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_451"
  op: "Identity"
  input: "save/RestoreV2:450"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_450"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel"
  input: "save/Identity_451"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_452"
  op: "Identity"
  input: "save/RestoreV2:451"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_451"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_m"
  input: "save/Identity_452"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_453"
  op: "Identity"
  input: "save/RestoreV2:452"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_452"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/intermediate/dense/kernel/adam_v"
  input: "save/Identity_453"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_454"
  op: "Identity"
  input: "save/RestoreV2:453"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_453"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta"
  input: "save/Identity_454"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_455"
  op: "Identity"
  input: "save/RestoreV2:454"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_454"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_m"
  input: "save/Identity_455"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_456"
  op: "Identity"
  input: "save/RestoreV2:455"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_455"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/beta/adam_v"
  input: "save/Identity_456"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_457"
  op: "Identity"
  input: "save/RestoreV2:456"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_456"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma"
  input: "save/Identity_457"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_458"
  op: "Identity"
  input: "save/RestoreV2:457"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_457"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_458"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_459"
  op: "Identity"
  input: "save/RestoreV2:458"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_458"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_459"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_460"
  op: "Identity"
  input: "save/RestoreV2:459"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_459"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias"
  input: "save/Identity_460"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_461"
  op: "Identity"
  input: "save/RestoreV2:460"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_460"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_m"
  input: "save/Identity_461"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_462"
  op: "Identity"
  input: "save/RestoreV2:461"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_461"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/bias/adam_v"
  input: "save/Identity_462"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_463"
  op: "Identity"
  input: "save/RestoreV2:462"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_462"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel"
  input: "save/Identity_463"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_464"
  op: "Identity"
  input: "save/RestoreV2:463"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_463"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_m"
  input: "save/Identity_464"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_465"
  op: "Identity"
  input: "save/RestoreV2:464"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_464"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_6/output/dense/kernel/adam_v"
  input: "save/Identity_465"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_466"
  op: "Identity"
  input: "save/RestoreV2:465"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_465"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta"
  input: "save/Identity_466"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_467"
  op: "Identity"
  input: "save/RestoreV2:466"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_466"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_467"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_468"
  op: "Identity"
  input: "save/RestoreV2:467"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_467"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_468"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_469"
  op: "Identity"
  input: "save/RestoreV2:468"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_468"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma"
  input: "save/Identity_469"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_470"
  op: "Identity"
  input: "save/RestoreV2:469"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_469"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_470"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_471"
  op: "Identity"
  input: "save/RestoreV2:470"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_470"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_471"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_472"
  op: "Identity"
  input: "save/RestoreV2:471"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_471"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias"
  input: "save/Identity_472"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_473"
  op: "Identity"
  input: "save/RestoreV2:472"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_472"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_m"
  input: "save/Identity_473"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_474"
  op: "Identity"
  input: "save/RestoreV2:473"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_473"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/bias/adam_v"
  input: "save/Identity_474"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_475"
  op: "Identity"
  input: "save/RestoreV2:474"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_474"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel"
  input: "save/Identity_475"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_476"
  op: "Identity"
  input: "save/RestoreV2:475"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_475"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_m"
  input: "save/Identity_476"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_477"
  op: "Identity"
  input: "save/RestoreV2:476"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_476"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/output/dense/kernel/adam_v"
  input: "save/Identity_477"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_478"
  op: "Identity"
  input: "save/RestoreV2:477"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_477"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias"
  input: "save/Identity_478"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_479"
  op: "Identity"
  input: "save/RestoreV2:478"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_478"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_m"
  input: "save/Identity_479"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_480"
  op: "Identity"
  input: "save/RestoreV2:479"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_479"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/bias/adam_v"
  input: "save/Identity_480"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_481"
  op: "Identity"
  input: "save/RestoreV2:480"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_480"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel"
  input: "save/Identity_481"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_482"
  op: "Identity"
  input: "save/RestoreV2:481"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_481"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_m"
  input: "save/Identity_482"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_483"
  op: "Identity"
  input: "save/RestoreV2:482"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_482"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/key/kernel/adam_v"
  input: "save/Identity_483"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_484"
  op: "Identity"
  input: "save/RestoreV2:483"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_483"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias"
  input: "save/Identity_484"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_485"
  op: "Identity"
  input: "save/RestoreV2:484"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_484"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_m"
  input: "save/Identity_485"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_486"
  op: "Identity"
  input: "save/RestoreV2:485"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_485"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/bias/adam_v"
  input: "save/Identity_486"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_487"
  op: "Identity"
  input: "save/RestoreV2:486"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_486"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel"
  input: "save/Identity_487"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_488"
  op: "Identity"
  input: "save/RestoreV2:487"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_487"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_m"
  input: "save/Identity_488"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_489"
  op: "Identity"
  input: "save/RestoreV2:488"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_488"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/query/kernel/adam_v"
  input: "save/Identity_489"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_490"
  op: "Identity"
  input: "save/RestoreV2:489"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_489"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias"
  input: "save/Identity_490"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_491"
  op: "Identity"
  input: "save/RestoreV2:490"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_490"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_m"
  input: "save/Identity_491"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_492"
  op: "Identity"
  input: "save/RestoreV2:491"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_491"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/bias/adam_v"
  input: "save/Identity_492"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_493"
  op: "Identity"
  input: "save/RestoreV2:492"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_492"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel"
  input: "save/Identity_493"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_494"
  op: "Identity"
  input: "save/RestoreV2:493"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_493"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_m"
  input: "save/Identity_494"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_495"
  op: "Identity"
  input: "save/RestoreV2:494"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_494"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/attention/self/value/kernel/adam_v"
  input: "save/Identity_495"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_496"
  op: "Identity"
  input: "save/RestoreV2:495"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_495"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias"
  input: "save/Identity_496"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_497"
  op: "Identity"
  input: "save/RestoreV2:496"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_496"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_m"
  input: "save/Identity_497"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_498"
  op: "Identity"
  input: "save/RestoreV2:497"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_497"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/bias/adam_v"
  input: "save/Identity_498"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_499"
  op: "Identity"
  input: "save/RestoreV2:498"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_498"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel"
  input: "save/Identity_499"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_500"
  op: "Identity"
  input: "save/RestoreV2:499"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_499"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_m"
  input: "save/Identity_500"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_501"
  op: "Identity"
  input: "save/RestoreV2:500"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_500"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/intermediate/dense/kernel/adam_v"
  input: "save/Identity_501"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_502"
  op: "Identity"
  input: "save/RestoreV2:501"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_501"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta"
  input: "save/Identity_502"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_503"
  op: "Identity"
  input: "save/RestoreV2:502"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_502"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_m"
  input: "save/Identity_503"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_504"
  op: "Identity"
  input: "save/RestoreV2:503"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_503"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/beta/adam_v"
  input: "save/Identity_504"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_505"
  op: "Identity"
  input: "save/RestoreV2:504"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_504"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma"
  input: "save/Identity_505"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_506"
  op: "Identity"
  input: "save/RestoreV2:505"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_505"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_506"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_507"
  op: "Identity"
  input: "save/RestoreV2:506"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_506"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_507"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_508"
  op: "Identity"
  input: "save/RestoreV2:507"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_507"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias"
  input: "save/Identity_508"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_509"
  op: "Identity"
  input: "save/RestoreV2:508"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_508"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_m"
  input: "save/Identity_509"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_510"
  op: "Identity"
  input: "save/RestoreV2:509"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_509"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/bias/adam_v"
  input: "save/Identity_510"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_511"
  op: "Identity"
  input: "save/RestoreV2:510"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_510"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel"
  input: "save/Identity_511"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_512"
  op: "Identity"
  input: "save/RestoreV2:511"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_511"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_m"
  input: "save/Identity_512"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_513"
  op: "Identity"
  input: "save/RestoreV2:512"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_512"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_7/output/dense/kernel/adam_v"
  input: "save/Identity_513"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_514"
  op: "Identity"
  input: "save/RestoreV2:513"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_513"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta"
  input: "save/Identity_514"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_515"
  op: "Identity"
  input: "save/RestoreV2:514"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_514"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_515"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_516"
  op: "Identity"
  input: "save/RestoreV2:515"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_515"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_516"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_517"
  op: "Identity"
  input: "save/RestoreV2:516"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_516"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma"
  input: "save/Identity_517"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_518"
  op: "Identity"
  input: "save/RestoreV2:517"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_517"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_518"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_519"
  op: "Identity"
  input: "save/RestoreV2:518"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_518"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_519"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_520"
  op: "Identity"
  input: "save/RestoreV2:519"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_519"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias"
  input: "save/Identity_520"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_521"
  op: "Identity"
  input: "save/RestoreV2:520"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_520"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_m"
  input: "save/Identity_521"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_522"
  op: "Identity"
  input: "save/RestoreV2:521"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_521"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/bias/adam_v"
  input: "save/Identity_522"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_523"
  op: "Identity"
  input: "save/RestoreV2:522"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_522"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel"
  input: "save/Identity_523"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_524"
  op: "Identity"
  input: "save/RestoreV2:523"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_523"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_m"
  input: "save/Identity_524"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_525"
  op: "Identity"
  input: "save/RestoreV2:524"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_524"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/output/dense/kernel/adam_v"
  input: "save/Identity_525"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_526"
  op: "Identity"
  input: "save/RestoreV2:525"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_525"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias"
  input: "save/Identity_526"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_527"
  op: "Identity"
  input: "save/RestoreV2:526"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_526"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_m"
  input: "save/Identity_527"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_528"
  op: "Identity"
  input: "save/RestoreV2:527"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_527"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/bias/adam_v"
  input: "save/Identity_528"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_529"
  op: "Identity"
  input: "save/RestoreV2:528"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_528"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel"
  input: "save/Identity_529"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_530"
  op: "Identity"
  input: "save/RestoreV2:529"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_529"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_m"
  input: "save/Identity_530"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_531"
  op: "Identity"
  input: "save/RestoreV2:530"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_530"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/key/kernel/adam_v"
  input: "save/Identity_531"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_532"
  op: "Identity"
  input: "save/RestoreV2:531"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_531"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias"
  input: "save/Identity_532"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_533"
  op: "Identity"
  input: "save/RestoreV2:532"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_532"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_m"
  input: "save/Identity_533"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_534"
  op: "Identity"
  input: "save/RestoreV2:533"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_533"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/bias/adam_v"
  input: "save/Identity_534"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_535"
  op: "Identity"
  input: "save/RestoreV2:534"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_534"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel"
  input: "save/Identity_535"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_536"
  op: "Identity"
  input: "save/RestoreV2:535"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_535"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_m"
  input: "save/Identity_536"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_537"
  op: "Identity"
  input: "save/RestoreV2:536"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_536"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/query/kernel/adam_v"
  input: "save/Identity_537"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_538"
  op: "Identity"
  input: "save/RestoreV2:537"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_537"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias"
  input: "save/Identity_538"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_539"
  op: "Identity"
  input: "save/RestoreV2:538"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_538"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_m"
  input: "save/Identity_539"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_540"
  op: "Identity"
  input: "save/RestoreV2:539"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_539"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/bias/adam_v"
  input: "save/Identity_540"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_541"
  op: "Identity"
  input: "save/RestoreV2:540"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_540"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel"
  input: "save/Identity_541"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_542"
  op: "Identity"
  input: "save/RestoreV2:541"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_541"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_m"
  input: "save/Identity_542"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_543"
  op: "Identity"
  input: "save/RestoreV2:542"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_542"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/attention/self/value/kernel/adam_v"
  input: "save/Identity_543"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_544"
  op: "Identity"
  input: "save/RestoreV2:543"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_543"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias"
  input: "save/Identity_544"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_545"
  op: "Identity"
  input: "save/RestoreV2:544"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_544"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_m"
  input: "save/Identity_545"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_546"
  op: "Identity"
  input: "save/RestoreV2:545"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_545"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/bias/adam_v"
  input: "save/Identity_546"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_547"
  op: "Identity"
  input: "save/RestoreV2:546"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_546"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel"
  input: "save/Identity_547"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_548"
  op: "Identity"
  input: "save/RestoreV2:547"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_547"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_m"
  input: "save/Identity_548"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_549"
  op: "Identity"
  input: "save/RestoreV2:548"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_548"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/intermediate/dense/kernel/adam_v"
  input: "save/Identity_549"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_550"
  op: "Identity"
  input: "save/RestoreV2:549"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_549"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta"
  input: "save/Identity_550"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_551"
  op: "Identity"
  input: "save/RestoreV2:550"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_550"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_m"
  input: "save/Identity_551"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_552"
  op: "Identity"
  input: "save/RestoreV2:551"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_551"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/beta/adam_v"
  input: "save/Identity_552"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_553"
  op: "Identity"
  input: "save/RestoreV2:552"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_552"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma"
  input: "save/Identity_553"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_554"
  op: "Identity"
  input: "save/RestoreV2:553"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_553"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_554"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_555"
  op: "Identity"
  input: "save/RestoreV2:554"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_554"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_555"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_556"
  op: "Identity"
  input: "save/RestoreV2:555"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_555"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias"
  input: "save/Identity_556"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_557"
  op: "Identity"
  input: "save/RestoreV2:556"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_556"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_m"
  input: "save/Identity_557"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_558"
  op: "Identity"
  input: "save/RestoreV2:557"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_557"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/bias/adam_v"
  input: "save/Identity_558"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_559"
  op: "Identity"
  input: "save/RestoreV2:558"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_558"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel"
  input: "save/Identity_559"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_560"
  op: "Identity"
  input: "save/RestoreV2:559"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_559"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_m"
  input: "save/Identity_560"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_561"
  op: "Identity"
  input: "save/RestoreV2:560"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_560"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_8/output/dense/kernel/adam_v"
  input: "save/Identity_561"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_562"
  op: "Identity"
  input: "save/RestoreV2:561"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_561"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta"
  input: "save/Identity_562"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_563"
  op: "Identity"
  input: "save/RestoreV2:562"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_562"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_563"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_564"
  op: "Identity"
  input: "save/RestoreV2:563"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_563"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_564"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_565"
  op: "Identity"
  input: "save/RestoreV2:564"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_564"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma"
  input: "save/Identity_565"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_566"
  op: "Identity"
  input: "save/RestoreV2:565"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_565"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_566"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_567"
  op: "Identity"
  input: "save/RestoreV2:566"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_566"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_567"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_568"
  op: "Identity"
  input: "save/RestoreV2:567"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_567"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias"
  input: "save/Identity_568"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_569"
  op: "Identity"
  input: "save/RestoreV2:568"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_568"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_m"
  input: "save/Identity_569"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_570"
  op: "Identity"
  input: "save/RestoreV2:569"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_569"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/bias/adam_v"
  input: "save/Identity_570"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_571"
  op: "Identity"
  input: "save/RestoreV2:570"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_570"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel"
  input: "save/Identity_571"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_572"
  op: "Identity"
  input: "save/RestoreV2:571"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_571"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_m"
  input: "save/Identity_572"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_573"
  op: "Identity"
  input: "save/RestoreV2:572"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_572"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/output/dense/kernel/adam_v"
  input: "save/Identity_573"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_574"
  op: "Identity"
  input: "save/RestoreV2:573"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_573"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias"
  input: "save/Identity_574"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_575"
  op: "Identity"
  input: "save/RestoreV2:574"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_574"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_m"
  input: "save/Identity_575"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_576"
  op: "Identity"
  input: "save/RestoreV2:575"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_575"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/bias/adam_v"
  input: "save/Identity_576"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_577"
  op: "Identity"
  input: "save/RestoreV2:576"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_576"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel"
  input: "save/Identity_577"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_578"
  op: "Identity"
  input: "save/RestoreV2:577"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_577"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_m"
  input: "save/Identity_578"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_579"
  op: "Identity"
  input: "save/RestoreV2:578"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_578"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/key/kernel/adam_v"
  input: "save/Identity_579"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_580"
  op: "Identity"
  input: "save/RestoreV2:579"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_579"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias"
  input: "save/Identity_580"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_581"
  op: "Identity"
  input: "save/RestoreV2:580"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_580"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_m"
  input: "save/Identity_581"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_582"
  op: "Identity"
  input: "save/RestoreV2:581"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_581"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/bias/adam_v"
  input: "save/Identity_582"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_583"
  op: "Identity"
  input: "save/RestoreV2:582"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_582"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel"
  input: "save/Identity_583"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_584"
  op: "Identity"
  input: "save/RestoreV2:583"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_583"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_m"
  input: "save/Identity_584"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_585"
  op: "Identity"
  input: "save/RestoreV2:584"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_584"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/query/kernel/adam_v"
  input: "save/Identity_585"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_586"
  op: "Identity"
  input: "save/RestoreV2:585"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_585"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias"
  input: "save/Identity_586"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_587"
  op: "Identity"
  input: "save/RestoreV2:586"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_586"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_m"
  input: "save/Identity_587"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_588"
  op: "Identity"
  input: "save/RestoreV2:587"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_587"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/bias/adam_v"
  input: "save/Identity_588"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_589"
  op: "Identity"
  input: "save/RestoreV2:588"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_588"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel"
  input: "save/Identity_589"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_590"
  op: "Identity"
  input: "save/RestoreV2:589"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_589"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_m"
  input: "save/Identity_590"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_591"
  op: "Identity"
  input: "save/RestoreV2:590"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_590"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/attention/self/value/kernel/adam_v"
  input: "save/Identity_591"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_592"
  op: "Identity"
  input: "save/RestoreV2:591"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_591"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias"
  input: "save/Identity_592"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_593"
  op: "Identity"
  input: "save/RestoreV2:592"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_592"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_m"
  input: "save/Identity_593"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_594"
  op: "Identity"
  input: "save/RestoreV2:593"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_593"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/bias/adam_v"
  input: "save/Identity_594"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_595"
  op: "Identity"
  input: "save/RestoreV2:594"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_594"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel"
  input: "save/Identity_595"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_596"
  op: "Identity"
  input: "save/RestoreV2:595"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_595"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_m"
  input: "save/Identity_596"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_597"
  op: "Identity"
  input: "save/RestoreV2:596"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_596"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/intermediate/dense/kernel/adam_v"
  input: "save/Identity_597"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_598"
  op: "Identity"
  input: "save/RestoreV2:597"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_597"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta"
  input: "save/Identity_598"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_599"
  op: "Identity"
  input: "save/RestoreV2:598"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_598"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_m"
  input: "save/Identity_599"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_600"
  op: "Identity"
  input: "save/RestoreV2:599"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_599"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/beta/adam_v"
  input: "save/Identity_600"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_601"
  op: "Identity"
  input: "save/RestoreV2:600"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_600"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma"
  input: "save/Identity_601"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_602"
  op: "Identity"
  input: "save/RestoreV2:601"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_601"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_602"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_603"
  op: "Identity"
  input: "save/RestoreV2:602"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_602"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_603"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_604"
  op: "Identity"
  input: "save/RestoreV2:603"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_603"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias"
  input: "save/Identity_604"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_605"
  op: "Identity"
  input: "save/RestoreV2:604"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_604"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_m"
  input: "save/Identity_605"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_606"
  op: "Identity"
  input: "save/RestoreV2:605"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_605"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/bias/adam_v"
  input: "save/Identity_606"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_607"
  op: "Identity"
  input: "save/RestoreV2:606"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_606"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel"
  input: "save/Identity_607"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_608"
  op: "Identity"
  input: "save/RestoreV2:607"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_607"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_m"
  input: "save/Identity_608"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_609"
  op: "Identity"
  input: "save/RestoreV2:608"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_608"
  op: "AssignVariableOp"
  input: "electra/encoder/layer_9/output/dense/kernel/adam_v"
  input: "save/Identity_609"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_610"
  op: "Identity"
  input: "save/RestoreV2:609"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_609"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/bias"
  input: "save/Identity_610"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_611"
  op: "Identity"
  input: "save/RestoreV2:610"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_610"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/bias/adam_m"
  input: "save/Identity_611"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_612"
  op: "Identity"
  input: "save/RestoreV2:611"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_611"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/bias/adam_v"
  input: "save/Identity_612"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_613"
  op: "Identity"
  input: "save/RestoreV2:612"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_612"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/kernel"
  input: "save/Identity_613"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_614"
  op: "Identity"
  input: "save/RestoreV2:613"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_613"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/kernel/adam_m"
  input: "save/Identity_614"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_615"
  op: "Identity"
  input: "save/RestoreV2:614"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_614"
  op: "AssignVariableOp"
  input: "generator/embeddings_project/kernel/adam_v"
  input: "save/Identity_615"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_616"
  op: "Identity"
  input: "save/RestoreV2:615"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_615"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta"
  input: "save/Identity_616"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_617"
  op: "Identity"
  input: "save/RestoreV2:616"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_616"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_617"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_618"
  op: "Identity"
  input: "save/RestoreV2:617"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_617"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_618"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_619"
  op: "Identity"
  input: "save/RestoreV2:618"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_618"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma"
  input: "save/Identity_619"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_620"
  op: "Identity"
  input: "save/RestoreV2:619"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_619"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_620"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_621"
  op: "Identity"
  input: "save/RestoreV2:620"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_620"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_621"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_622"
  op: "Identity"
  input: "save/RestoreV2:621"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_621"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias"
  input: "save/Identity_622"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_623"
  op: "Identity"
  input: "save/RestoreV2:622"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_622"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_m"
  input: "save/Identity_623"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_624"
  op: "Identity"
  input: "save/RestoreV2:623"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_623"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/bias/adam_v"
  input: "save/Identity_624"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_625"
  op: "Identity"
  input: "save/RestoreV2:624"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_624"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel"
  input: "save/Identity_625"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_626"
  op: "Identity"
  input: "save/RestoreV2:625"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_625"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_m"
  input: "save/Identity_626"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_627"
  op: "Identity"
  input: "save/RestoreV2:626"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_626"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/output/dense/kernel/adam_v"
  input: "save/Identity_627"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_628"
  op: "Identity"
  input: "save/RestoreV2:627"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_627"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias"
  input: "save/Identity_628"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_629"
  op: "Identity"
  input: "save/RestoreV2:628"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_628"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_m"
  input: "save/Identity_629"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_630"
  op: "Identity"
  input: "save/RestoreV2:629"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_629"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/bias/adam_v"
  input: "save/Identity_630"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_631"
  op: "Identity"
  input: "save/RestoreV2:630"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_630"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel"
  input: "save/Identity_631"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_632"
  op: "Identity"
  input: "save/RestoreV2:631"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_631"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_m"
  input: "save/Identity_632"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_633"
  op: "Identity"
  input: "save/RestoreV2:632"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_632"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/key/kernel/adam_v"
  input: "save/Identity_633"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_634"
  op: "Identity"
  input: "save/RestoreV2:633"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_633"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias"
  input: "save/Identity_634"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_635"
  op: "Identity"
  input: "save/RestoreV2:634"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_634"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_m"
  input: "save/Identity_635"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_636"
  op: "Identity"
  input: "save/RestoreV2:635"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_635"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/bias/adam_v"
  input: "save/Identity_636"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_637"
  op: "Identity"
  input: "save/RestoreV2:636"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_636"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel"
  input: "save/Identity_637"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_638"
  op: "Identity"
  input: "save/RestoreV2:637"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_637"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_m"
  input: "save/Identity_638"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_639"
  op: "Identity"
  input: "save/RestoreV2:638"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_638"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/query/kernel/adam_v"
  input: "save/Identity_639"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_640"
  op: "Identity"
  input: "save/RestoreV2:639"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_639"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias"
  input: "save/Identity_640"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_641"
  op: "Identity"
  input: "save/RestoreV2:640"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_640"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_m"
  input: "save/Identity_641"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_642"
  op: "Identity"
  input: "save/RestoreV2:641"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_641"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/bias/adam_v"
  input: "save/Identity_642"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_643"
  op: "Identity"
  input: "save/RestoreV2:642"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_642"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel"
  input: "save/Identity_643"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_644"
  op: "Identity"
  input: "save/RestoreV2:643"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_643"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_m"
  input: "save/Identity_644"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_645"
  op: "Identity"
  input: "save/RestoreV2:644"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_644"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/attention/self/value/kernel/adam_v"
  input: "save/Identity_645"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_646"
  op: "Identity"
  input: "save/RestoreV2:645"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_645"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias"
  input: "save/Identity_646"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_647"
  op: "Identity"
  input: "save/RestoreV2:646"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_646"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_m"
  input: "save/Identity_647"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_648"
  op: "Identity"
  input: "save/RestoreV2:647"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_647"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/bias/adam_v"
  input: "save/Identity_648"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_649"
  op: "Identity"
  input: "save/RestoreV2:648"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_648"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel"
  input: "save/Identity_649"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_650"
  op: "Identity"
  input: "save/RestoreV2:649"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_649"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_m"
  input: "save/Identity_650"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_651"
  op: "Identity"
  input: "save/RestoreV2:650"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_650"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/intermediate/dense/kernel/adam_v"
  input: "save/Identity_651"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_652"
  op: "Identity"
  input: "save/RestoreV2:651"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_651"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta"
  input: "save/Identity_652"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_653"
  op: "Identity"
  input: "save/RestoreV2:652"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_652"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_m"
  input: "save/Identity_653"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_654"
  op: "Identity"
  input: "save/RestoreV2:653"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_653"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/beta/adam_v"
  input: "save/Identity_654"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_655"
  op: "Identity"
  input: "save/RestoreV2:654"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_654"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma"
  input: "save/Identity_655"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_656"
  op: "Identity"
  input: "save/RestoreV2:655"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_655"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_656"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_657"
  op: "Identity"
  input: "save/RestoreV2:656"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_656"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_657"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_658"
  op: "Identity"
  input: "save/RestoreV2:657"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_657"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias"
  input: "save/Identity_658"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_659"
  op: "Identity"
  input: "save/RestoreV2:658"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_658"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_m"
  input: "save/Identity_659"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_660"
  op: "Identity"
  input: "save/RestoreV2:659"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_659"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/bias/adam_v"
  input: "save/Identity_660"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_661"
  op: "Identity"
  input: "save/RestoreV2:660"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_660"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel"
  input: "save/Identity_661"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_662"
  op: "Identity"
  input: "save/RestoreV2:661"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_661"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_m"
  input: "save/Identity_662"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_663"
  op: "Identity"
  input: "save/RestoreV2:662"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_662"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_0/output/dense/kernel/adam_v"
  input: "save/Identity_663"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_664"
  op: "Identity"
  input: "save/RestoreV2:663"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_663"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta"
  input: "save/Identity_664"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_665"
  op: "Identity"
  input: "save/RestoreV2:664"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_664"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_665"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_666"
  op: "Identity"
  input: "save/RestoreV2:665"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_665"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_666"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_667"
  op: "Identity"
  input: "save/RestoreV2:666"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_666"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma"
  input: "save/Identity_667"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_668"
  op: "Identity"
  input: "save/RestoreV2:667"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_667"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_668"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_669"
  op: "Identity"
  input: "save/RestoreV2:668"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_668"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_669"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_670"
  op: "Identity"
  input: "save/RestoreV2:669"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_669"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias"
  input: "save/Identity_670"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_671"
  op: "Identity"
  input: "save/RestoreV2:670"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_670"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_m"
  input: "save/Identity_671"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_672"
  op: "Identity"
  input: "save/RestoreV2:671"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_671"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/bias/adam_v"
  input: "save/Identity_672"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_673"
  op: "Identity"
  input: "save/RestoreV2:672"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_672"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel"
  input: "save/Identity_673"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_674"
  op: "Identity"
  input: "save/RestoreV2:673"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_673"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_m"
  input: "save/Identity_674"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_675"
  op: "Identity"
  input: "save/RestoreV2:674"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_674"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/output/dense/kernel/adam_v"
  input: "save/Identity_675"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_676"
  op: "Identity"
  input: "save/RestoreV2:675"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_675"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias"
  input: "save/Identity_676"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_677"
  op: "Identity"
  input: "save/RestoreV2:676"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_676"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_m"
  input: "save/Identity_677"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_678"
  op: "Identity"
  input: "save/RestoreV2:677"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_677"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/bias/adam_v"
  input: "save/Identity_678"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_679"
  op: "Identity"
  input: "save/RestoreV2:678"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_678"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel"
  input: "save/Identity_679"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_680"
  op: "Identity"
  input: "save/RestoreV2:679"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_679"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_m"
  input: "save/Identity_680"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_681"
  op: "Identity"
  input: "save/RestoreV2:680"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_680"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/key/kernel/adam_v"
  input: "save/Identity_681"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_682"
  op: "Identity"
  input: "save/RestoreV2:681"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_681"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias"
  input: "save/Identity_682"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_683"
  op: "Identity"
  input: "save/RestoreV2:682"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_682"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_m"
  input: "save/Identity_683"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_684"
  op: "Identity"
  input: "save/RestoreV2:683"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_683"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/bias/adam_v"
  input: "save/Identity_684"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_685"
  op: "Identity"
  input: "save/RestoreV2:684"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_684"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel"
  input: "save/Identity_685"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_686"
  op: "Identity"
  input: "save/RestoreV2:685"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_685"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_m"
  input: "save/Identity_686"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_687"
  op: "Identity"
  input: "save/RestoreV2:686"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_686"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/query/kernel/adam_v"
  input: "save/Identity_687"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_688"
  op: "Identity"
  input: "save/RestoreV2:687"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_687"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias"
  input: "save/Identity_688"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_689"
  op: "Identity"
  input: "save/RestoreV2:688"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_688"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_m"
  input: "save/Identity_689"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_690"
  op: "Identity"
  input: "save/RestoreV2:689"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_689"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/bias/adam_v"
  input: "save/Identity_690"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_691"
  op: "Identity"
  input: "save/RestoreV2:690"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_690"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel"
  input: "save/Identity_691"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_692"
  op: "Identity"
  input: "save/RestoreV2:691"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_691"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_m"
  input: "save/Identity_692"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_693"
  op: "Identity"
  input: "save/RestoreV2:692"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_692"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/attention/self/value/kernel/adam_v"
  input: "save/Identity_693"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_694"
  op: "Identity"
  input: "save/RestoreV2:693"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_693"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias"
  input: "save/Identity_694"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_695"
  op: "Identity"
  input: "save/RestoreV2:694"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_694"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_m"
  input: "save/Identity_695"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_696"
  op: "Identity"
  input: "save/RestoreV2:695"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_695"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/bias/adam_v"
  input: "save/Identity_696"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_697"
  op: "Identity"
  input: "save/RestoreV2:696"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_696"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel"
  input: "save/Identity_697"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_698"
  op: "Identity"
  input: "save/RestoreV2:697"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_697"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_m"
  input: "save/Identity_698"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_699"
  op: "Identity"
  input: "save/RestoreV2:698"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_698"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/intermediate/dense/kernel/adam_v"
  input: "save/Identity_699"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_700"
  op: "Identity"
  input: "save/RestoreV2:699"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_699"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta"
  input: "save/Identity_700"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_701"
  op: "Identity"
  input: "save/RestoreV2:700"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_700"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_m"
  input: "save/Identity_701"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_702"
  op: "Identity"
  input: "save/RestoreV2:701"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_701"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/beta/adam_v"
  input: "save/Identity_702"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_703"
  op: "Identity"
  input: "save/RestoreV2:702"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_702"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma"
  input: "save/Identity_703"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_704"
  op: "Identity"
  input: "save/RestoreV2:703"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_703"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_704"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_705"
  op: "Identity"
  input: "save/RestoreV2:704"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_704"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_705"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_706"
  op: "Identity"
  input: "save/RestoreV2:705"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_705"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias"
  input: "save/Identity_706"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_707"
  op: "Identity"
  input: "save/RestoreV2:706"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_706"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_m"
  input: "save/Identity_707"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_708"
  op: "Identity"
  input: "save/RestoreV2:707"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_707"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/bias/adam_v"
  input: "save/Identity_708"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_709"
  op: "Identity"
  input: "save/RestoreV2:708"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_708"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel"
  input: "save/Identity_709"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_710"
  op: "Identity"
  input: "save/RestoreV2:709"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_709"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_m"
  input: "save/Identity_710"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_711"
  op: "Identity"
  input: "save/RestoreV2:710"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_710"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_1/output/dense/kernel/adam_v"
  input: "save/Identity_711"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_712"
  op: "Identity"
  input: "save/RestoreV2:711"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_711"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta"
  input: "save/Identity_712"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_713"
  op: "Identity"
  input: "save/RestoreV2:712"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_712"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_713"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_714"
  op: "Identity"
  input: "save/RestoreV2:713"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_713"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_714"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_715"
  op: "Identity"
  input: "save/RestoreV2:714"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_714"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma"
  input: "save/Identity_715"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_716"
  op: "Identity"
  input: "save/RestoreV2:715"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_715"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_716"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_717"
  op: "Identity"
  input: "save/RestoreV2:716"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_716"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_717"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_718"
  op: "Identity"
  input: "save/RestoreV2:717"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_717"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias"
  input: "save/Identity_718"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_719"
  op: "Identity"
  input: "save/RestoreV2:718"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_718"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_m"
  input: "save/Identity_719"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_720"
  op: "Identity"
  input: "save/RestoreV2:719"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_719"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/bias/adam_v"
  input: "save/Identity_720"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_721"
  op: "Identity"
  input: "save/RestoreV2:720"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_720"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel"
  input: "save/Identity_721"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_722"
  op: "Identity"
  input: "save/RestoreV2:721"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_721"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_m"
  input: "save/Identity_722"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_723"
  op: "Identity"
  input: "save/RestoreV2:722"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_722"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/output/dense/kernel/adam_v"
  input: "save/Identity_723"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_724"
  op: "Identity"
  input: "save/RestoreV2:723"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_723"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias"
  input: "save/Identity_724"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_725"
  op: "Identity"
  input: "save/RestoreV2:724"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_724"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_m"
  input: "save/Identity_725"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_726"
  op: "Identity"
  input: "save/RestoreV2:725"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_725"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/bias/adam_v"
  input: "save/Identity_726"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_727"
  op: "Identity"
  input: "save/RestoreV2:726"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_726"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel"
  input: "save/Identity_727"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_728"
  op: "Identity"
  input: "save/RestoreV2:727"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_727"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_m"
  input: "save/Identity_728"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_729"
  op: "Identity"
  input: "save/RestoreV2:728"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_728"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/key/kernel/adam_v"
  input: "save/Identity_729"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_730"
  op: "Identity"
  input: "save/RestoreV2:729"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_729"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias"
  input: "save/Identity_730"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_731"
  op: "Identity"
  input: "save/RestoreV2:730"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_730"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_m"
  input: "save/Identity_731"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_732"
  op: "Identity"
  input: "save/RestoreV2:731"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_731"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/bias/adam_v"
  input: "save/Identity_732"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_733"
  op: "Identity"
  input: "save/RestoreV2:732"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_732"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel"
  input: "save/Identity_733"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_734"
  op: "Identity"
  input: "save/RestoreV2:733"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_733"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_m"
  input: "save/Identity_734"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_735"
  op: "Identity"
  input: "save/RestoreV2:734"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_734"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/query/kernel/adam_v"
  input: "save/Identity_735"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_736"
  op: "Identity"
  input: "save/RestoreV2:735"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_735"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias"
  input: "save/Identity_736"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_737"
  op: "Identity"
  input: "save/RestoreV2:736"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_736"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_m"
  input: "save/Identity_737"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_738"
  op: "Identity"
  input: "save/RestoreV2:737"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_737"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/bias/adam_v"
  input: "save/Identity_738"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_739"
  op: "Identity"
  input: "save/RestoreV2:738"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_738"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel"
  input: "save/Identity_739"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_740"
  op: "Identity"
  input: "save/RestoreV2:739"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_739"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_m"
  input: "save/Identity_740"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_741"
  op: "Identity"
  input: "save/RestoreV2:740"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_740"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/attention/self/value/kernel/adam_v"
  input: "save/Identity_741"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_742"
  op: "Identity"
  input: "save/RestoreV2:741"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_741"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias"
  input: "save/Identity_742"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_743"
  op: "Identity"
  input: "save/RestoreV2:742"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_742"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_m"
  input: "save/Identity_743"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_744"
  op: "Identity"
  input: "save/RestoreV2:743"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_743"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/bias/adam_v"
  input: "save/Identity_744"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_745"
  op: "Identity"
  input: "save/RestoreV2:744"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_744"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel"
  input: "save/Identity_745"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_746"
  op: "Identity"
  input: "save/RestoreV2:745"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_745"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_m"
  input: "save/Identity_746"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_747"
  op: "Identity"
  input: "save/RestoreV2:746"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_746"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/intermediate/dense/kernel/adam_v"
  input: "save/Identity_747"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_748"
  op: "Identity"
  input: "save/RestoreV2:747"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_747"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta"
  input: "save/Identity_748"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_749"
  op: "Identity"
  input: "save/RestoreV2:748"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_748"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_m"
  input: "save/Identity_749"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_750"
  op: "Identity"
  input: "save/RestoreV2:749"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_749"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/beta/adam_v"
  input: "save/Identity_750"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_751"
  op: "Identity"
  input: "save/RestoreV2:750"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_750"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma"
  input: "save/Identity_751"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_752"
  op: "Identity"
  input: "save/RestoreV2:751"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_751"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_752"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_753"
  op: "Identity"
  input: "save/RestoreV2:752"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_752"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_753"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_754"
  op: "Identity"
  input: "save/RestoreV2:753"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_753"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias"
  input: "save/Identity_754"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_755"
  op: "Identity"
  input: "save/RestoreV2:754"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_754"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_m"
  input: "save/Identity_755"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_756"
  op: "Identity"
  input: "save/RestoreV2:755"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_755"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/bias/adam_v"
  input: "save/Identity_756"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_757"
  op: "Identity"
  input: "save/RestoreV2:756"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_756"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel"
  input: "save/Identity_757"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_758"
  op: "Identity"
  input: "save/RestoreV2:757"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_757"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_m"
  input: "save/Identity_758"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_759"
  op: "Identity"
  input: "save/RestoreV2:758"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_758"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_10/output/dense/kernel/adam_v"
  input: "save/Identity_759"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_760"
  op: "Identity"
  input: "save/RestoreV2:759"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_759"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta"
  input: "save/Identity_760"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_761"
  op: "Identity"
  input: "save/RestoreV2:760"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_760"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_761"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_762"
  op: "Identity"
  input: "save/RestoreV2:761"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_761"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_762"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_763"
  op: "Identity"
  input: "save/RestoreV2:762"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_762"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma"
  input: "save/Identity_763"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_764"
  op: "Identity"
  input: "save/RestoreV2:763"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_763"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_764"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_765"
  op: "Identity"
  input: "save/RestoreV2:764"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_764"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_765"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_766"
  op: "Identity"
  input: "save/RestoreV2:765"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_765"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias"
  input: "save/Identity_766"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_767"
  op: "Identity"
  input: "save/RestoreV2:766"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_766"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_m"
  input: "save/Identity_767"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_768"
  op: "Identity"
  input: "save/RestoreV2:767"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_767"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/bias/adam_v"
  input: "save/Identity_768"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_769"
  op: "Identity"
  input: "save/RestoreV2:768"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_768"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel"
  input: "save/Identity_769"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_770"
  op: "Identity"
  input: "save/RestoreV2:769"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_769"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_m"
  input: "save/Identity_770"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_771"
  op: "Identity"
  input: "save/RestoreV2:770"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_770"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/output/dense/kernel/adam_v"
  input: "save/Identity_771"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_772"
  op: "Identity"
  input: "save/RestoreV2:771"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_771"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias"
  input: "save/Identity_772"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_773"
  op: "Identity"
  input: "save/RestoreV2:772"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_772"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_m"
  input: "save/Identity_773"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_774"
  op: "Identity"
  input: "save/RestoreV2:773"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_773"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/bias/adam_v"
  input: "save/Identity_774"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_775"
  op: "Identity"
  input: "save/RestoreV2:774"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_774"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel"
  input: "save/Identity_775"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_776"
  op: "Identity"
  input: "save/RestoreV2:775"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_775"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_m"
  input: "save/Identity_776"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_777"
  op: "Identity"
  input: "save/RestoreV2:776"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_776"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/key/kernel/adam_v"
  input: "save/Identity_777"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_778"
  op: "Identity"
  input: "save/RestoreV2:777"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_777"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias"
  input: "save/Identity_778"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_779"
  op: "Identity"
  input: "save/RestoreV2:778"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_778"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_m"
  input: "save/Identity_779"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_780"
  op: "Identity"
  input: "save/RestoreV2:779"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_779"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/bias/adam_v"
  input: "save/Identity_780"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_781"
  op: "Identity"
  input: "save/RestoreV2:780"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_780"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel"
  input: "save/Identity_781"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_782"
  op: "Identity"
  input: "save/RestoreV2:781"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_781"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_m"
  input: "save/Identity_782"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_783"
  op: "Identity"
  input: "save/RestoreV2:782"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_782"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/query/kernel/adam_v"
  input: "save/Identity_783"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_784"
  op: "Identity"
  input: "save/RestoreV2:783"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_783"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias"
  input: "save/Identity_784"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_785"
  op: "Identity"
  input: "save/RestoreV2:784"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_784"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_m"
  input: "save/Identity_785"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_786"
  op: "Identity"
  input: "save/RestoreV2:785"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_785"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/bias/adam_v"
  input: "save/Identity_786"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_787"
  op: "Identity"
  input: "save/RestoreV2:786"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_786"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel"
  input: "save/Identity_787"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_788"
  op: "Identity"
  input: "save/RestoreV2:787"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_787"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_m"
  input: "save/Identity_788"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_789"
  op: "Identity"
  input: "save/RestoreV2:788"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_788"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/attention/self/value/kernel/adam_v"
  input: "save/Identity_789"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_790"
  op: "Identity"
  input: "save/RestoreV2:789"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_789"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias"
  input: "save/Identity_790"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_791"
  op: "Identity"
  input: "save/RestoreV2:790"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_790"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_m"
  input: "save/Identity_791"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_792"
  op: "Identity"
  input: "save/RestoreV2:791"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_791"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/bias/adam_v"
  input: "save/Identity_792"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_793"
  op: "Identity"
  input: "save/RestoreV2:792"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_792"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel"
  input: "save/Identity_793"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_794"
  op: "Identity"
  input: "save/RestoreV2:793"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_793"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_m"
  input: "save/Identity_794"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_795"
  op: "Identity"
  input: "save/RestoreV2:794"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_794"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/intermediate/dense/kernel/adam_v"
  input: "save/Identity_795"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_796"
  op: "Identity"
  input: "save/RestoreV2:795"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_795"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta"
  input: "save/Identity_796"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_797"
  op: "Identity"
  input: "save/RestoreV2:796"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_796"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_m"
  input: "save/Identity_797"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_798"
  op: "Identity"
  input: "save/RestoreV2:797"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_797"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/beta/adam_v"
  input: "save/Identity_798"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_799"
  op: "Identity"
  input: "save/RestoreV2:798"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_798"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma"
  input: "save/Identity_799"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_800"
  op: "Identity"
  input: "save/RestoreV2:799"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_799"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_800"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_801"
  op: "Identity"
  input: "save/RestoreV2:800"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_800"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_801"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_802"
  op: "Identity"
  input: "save/RestoreV2:801"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_801"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias"
  input: "save/Identity_802"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_803"
  op: "Identity"
  input: "save/RestoreV2:802"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_802"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_m"
  input: "save/Identity_803"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_804"
  op: "Identity"
  input: "save/RestoreV2:803"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_803"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/bias/adam_v"
  input: "save/Identity_804"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_805"
  op: "Identity"
  input: "save/RestoreV2:804"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_804"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel"
  input: "save/Identity_805"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_806"
  op: "Identity"
  input: "save/RestoreV2:805"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_805"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_m"
  input: "save/Identity_806"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_807"
  op: "Identity"
  input: "save/RestoreV2:806"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_806"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_11/output/dense/kernel/adam_v"
  input: "save/Identity_807"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_808"
  op: "Identity"
  input: "save/RestoreV2:807"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_807"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta"
  input: "save/Identity_808"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_809"
  op: "Identity"
  input: "save/RestoreV2:808"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_808"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_809"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_810"
  op: "Identity"
  input: "save/RestoreV2:809"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_809"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_810"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_811"
  op: "Identity"
  input: "save/RestoreV2:810"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_810"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma"
  input: "save/Identity_811"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_812"
  op: "Identity"
  input: "save/RestoreV2:811"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_811"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_812"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_813"
  op: "Identity"
  input: "save/RestoreV2:812"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_812"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_813"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_814"
  op: "Identity"
  input: "save/RestoreV2:813"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_813"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias"
  input: "save/Identity_814"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_815"
  op: "Identity"
  input: "save/RestoreV2:814"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_814"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_m"
  input: "save/Identity_815"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_816"
  op: "Identity"
  input: "save/RestoreV2:815"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_815"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/bias/adam_v"
  input: "save/Identity_816"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_817"
  op: "Identity"
  input: "save/RestoreV2:816"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_816"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel"
  input: "save/Identity_817"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_818"
  op: "Identity"
  input: "save/RestoreV2:817"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_817"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_m"
  input: "save/Identity_818"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_819"
  op: "Identity"
  input: "save/RestoreV2:818"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_818"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/output/dense/kernel/adam_v"
  input: "save/Identity_819"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_820"
  op: "Identity"
  input: "save/RestoreV2:819"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_819"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias"
  input: "save/Identity_820"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_821"
  op: "Identity"
  input: "save/RestoreV2:820"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_820"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_m"
  input: "save/Identity_821"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_822"
  op: "Identity"
  input: "save/RestoreV2:821"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_821"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/bias/adam_v"
  input: "save/Identity_822"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_823"
  op: "Identity"
  input: "save/RestoreV2:822"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_822"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel"
  input: "save/Identity_823"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_824"
  op: "Identity"
  input: "save/RestoreV2:823"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_823"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_m"
  input: "save/Identity_824"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_825"
  op: "Identity"
  input: "save/RestoreV2:824"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_824"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/key/kernel/adam_v"
  input: "save/Identity_825"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_826"
  op: "Identity"
  input: "save/RestoreV2:825"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_825"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias"
  input: "save/Identity_826"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_827"
  op: "Identity"
  input: "save/RestoreV2:826"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_826"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_m"
  input: "save/Identity_827"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_828"
  op: "Identity"
  input: "save/RestoreV2:827"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_827"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/bias/adam_v"
  input: "save/Identity_828"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_829"
  op: "Identity"
  input: "save/RestoreV2:828"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_828"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel"
  input: "save/Identity_829"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_830"
  op: "Identity"
  input: "save/RestoreV2:829"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_829"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_m"
  input: "save/Identity_830"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_831"
  op: "Identity"
  input: "save/RestoreV2:830"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_830"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/query/kernel/adam_v"
  input: "save/Identity_831"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_832"
  op: "Identity"
  input: "save/RestoreV2:831"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_831"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias"
  input: "save/Identity_832"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_833"
  op: "Identity"
  input: "save/RestoreV2:832"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_832"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_m"
  input: "save/Identity_833"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_834"
  op: "Identity"
  input: "save/RestoreV2:833"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_833"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/bias/adam_v"
  input: "save/Identity_834"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_835"
  op: "Identity"
  input: "save/RestoreV2:834"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_834"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel"
  input: "save/Identity_835"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_836"
  op: "Identity"
  input: "save/RestoreV2:835"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_835"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_m"
  input: "save/Identity_836"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_837"
  op: "Identity"
  input: "save/RestoreV2:836"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_836"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/attention/self/value/kernel/adam_v"
  input: "save/Identity_837"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_838"
  op: "Identity"
  input: "save/RestoreV2:837"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_837"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias"
  input: "save/Identity_838"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_839"
  op: "Identity"
  input: "save/RestoreV2:838"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_838"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_m"
  input: "save/Identity_839"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_840"
  op: "Identity"
  input: "save/RestoreV2:839"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_839"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/bias/adam_v"
  input: "save/Identity_840"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_841"
  op: "Identity"
  input: "save/RestoreV2:840"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_840"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel"
  input: "save/Identity_841"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_842"
  op: "Identity"
  input: "save/RestoreV2:841"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_841"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_m"
  input: "save/Identity_842"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_843"
  op: "Identity"
  input: "save/RestoreV2:842"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_842"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/intermediate/dense/kernel/adam_v"
  input: "save/Identity_843"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_844"
  op: "Identity"
  input: "save/RestoreV2:843"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_843"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta"
  input: "save/Identity_844"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_845"
  op: "Identity"
  input: "save/RestoreV2:844"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_844"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_m"
  input: "save/Identity_845"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_846"
  op: "Identity"
  input: "save/RestoreV2:845"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_845"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/beta/adam_v"
  input: "save/Identity_846"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_847"
  op: "Identity"
  input: "save/RestoreV2:846"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_846"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma"
  input: "save/Identity_847"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_848"
  op: "Identity"
  input: "save/RestoreV2:847"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_847"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_848"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_849"
  op: "Identity"
  input: "save/RestoreV2:848"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_848"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_849"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_850"
  op: "Identity"
  input: "save/RestoreV2:849"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_849"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias"
  input: "save/Identity_850"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_851"
  op: "Identity"
  input: "save/RestoreV2:850"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_850"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_m"
  input: "save/Identity_851"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_852"
  op: "Identity"
  input: "save/RestoreV2:851"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_851"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/bias/adam_v"
  input: "save/Identity_852"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_853"
  op: "Identity"
  input: "save/RestoreV2:852"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_852"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel"
  input: "save/Identity_853"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_854"
  op: "Identity"
  input: "save/RestoreV2:853"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_853"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_m"
  input: "save/Identity_854"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_855"
  op: "Identity"
  input: "save/RestoreV2:854"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_854"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_2/output/dense/kernel/adam_v"
  input: "save/Identity_855"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_856"
  op: "Identity"
  input: "save/RestoreV2:855"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_855"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta"
  input: "save/Identity_856"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_857"
  op: "Identity"
  input: "save/RestoreV2:856"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_856"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_857"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_858"
  op: "Identity"
  input: "save/RestoreV2:857"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_857"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_858"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_859"
  op: "Identity"
  input: "save/RestoreV2:858"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_858"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma"
  input: "save/Identity_859"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_860"
  op: "Identity"
  input: "save/RestoreV2:859"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_859"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_860"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_861"
  op: "Identity"
  input: "save/RestoreV2:860"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_860"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_861"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_862"
  op: "Identity"
  input: "save/RestoreV2:861"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_861"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias"
  input: "save/Identity_862"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_863"
  op: "Identity"
  input: "save/RestoreV2:862"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_862"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_m"
  input: "save/Identity_863"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_864"
  op: "Identity"
  input: "save/RestoreV2:863"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_863"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/bias/adam_v"
  input: "save/Identity_864"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_865"
  op: "Identity"
  input: "save/RestoreV2:864"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_864"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel"
  input: "save/Identity_865"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_866"
  op: "Identity"
  input: "save/RestoreV2:865"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_865"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_m"
  input: "save/Identity_866"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_867"
  op: "Identity"
  input: "save/RestoreV2:866"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_866"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/output/dense/kernel/adam_v"
  input: "save/Identity_867"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_868"
  op: "Identity"
  input: "save/RestoreV2:867"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_867"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias"
  input: "save/Identity_868"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_869"
  op: "Identity"
  input: "save/RestoreV2:868"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_868"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_m"
  input: "save/Identity_869"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_870"
  op: "Identity"
  input: "save/RestoreV2:869"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_869"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/bias/adam_v"
  input: "save/Identity_870"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_871"
  op: "Identity"
  input: "save/RestoreV2:870"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_870"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel"
  input: "save/Identity_871"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_872"
  op: "Identity"
  input: "save/RestoreV2:871"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_871"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_m"
  input: "save/Identity_872"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_873"
  op: "Identity"
  input: "save/RestoreV2:872"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_872"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/key/kernel/adam_v"
  input: "save/Identity_873"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_874"
  op: "Identity"
  input: "save/RestoreV2:873"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_873"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias"
  input: "save/Identity_874"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_875"
  op: "Identity"
  input: "save/RestoreV2:874"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_874"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_m"
  input: "save/Identity_875"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_876"
  op: "Identity"
  input: "save/RestoreV2:875"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_875"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/bias/adam_v"
  input: "save/Identity_876"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_877"
  op: "Identity"
  input: "save/RestoreV2:876"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_876"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel"
  input: "save/Identity_877"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_878"
  op: "Identity"
  input: "save/RestoreV2:877"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_877"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_m"
  input: "save/Identity_878"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_879"
  op: "Identity"
  input: "save/RestoreV2:878"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_878"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/query/kernel/adam_v"
  input: "save/Identity_879"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_880"
  op: "Identity"
  input: "save/RestoreV2:879"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_879"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias"
  input: "save/Identity_880"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_881"
  op: "Identity"
  input: "save/RestoreV2:880"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_880"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_m"
  input: "save/Identity_881"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_882"
  op: "Identity"
  input: "save/RestoreV2:881"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_881"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/bias/adam_v"
  input: "save/Identity_882"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_883"
  op: "Identity"
  input: "save/RestoreV2:882"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_882"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel"
  input: "save/Identity_883"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_884"
  op: "Identity"
  input: "save/RestoreV2:883"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_883"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_m"
  input: "save/Identity_884"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_885"
  op: "Identity"
  input: "save/RestoreV2:884"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_884"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/attention/self/value/kernel/adam_v"
  input: "save/Identity_885"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_886"
  op: "Identity"
  input: "save/RestoreV2:885"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_885"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias"
  input: "save/Identity_886"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_887"
  op: "Identity"
  input: "save/RestoreV2:886"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_886"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_m"
  input: "save/Identity_887"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_888"
  op: "Identity"
  input: "save/RestoreV2:887"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_887"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/bias/adam_v"
  input: "save/Identity_888"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_889"
  op: "Identity"
  input: "save/RestoreV2:888"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_888"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel"
  input: "save/Identity_889"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_890"
  op: "Identity"
  input: "save/RestoreV2:889"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_889"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_m"
  input: "save/Identity_890"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_891"
  op: "Identity"
  input: "save/RestoreV2:890"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_890"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/intermediate/dense/kernel/adam_v"
  input: "save/Identity_891"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_892"
  op: "Identity"
  input: "save/RestoreV2:891"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_891"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta"
  input: "save/Identity_892"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_893"
  op: "Identity"
  input: "save/RestoreV2:892"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_892"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_m"
  input: "save/Identity_893"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_894"
  op: "Identity"
  input: "save/RestoreV2:893"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_893"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/beta/adam_v"
  input: "save/Identity_894"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_895"
  op: "Identity"
  input: "save/RestoreV2:894"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_894"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma"
  input: "save/Identity_895"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_896"
  op: "Identity"
  input: "save/RestoreV2:895"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_895"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_896"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_897"
  op: "Identity"
  input: "save/RestoreV2:896"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_896"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_897"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_898"
  op: "Identity"
  input: "save/RestoreV2:897"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_897"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias"
  input: "save/Identity_898"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_899"
  op: "Identity"
  input: "save/RestoreV2:898"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_898"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_m"
  input: "save/Identity_899"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_900"
  op: "Identity"
  input: "save/RestoreV2:899"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_899"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/bias/adam_v"
  input: "save/Identity_900"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_901"
  op: "Identity"
  input: "save/RestoreV2:900"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_900"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel"
  input: "save/Identity_901"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_902"
  op: "Identity"
  input: "save/RestoreV2:901"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_901"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_m"
  input: "save/Identity_902"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_903"
  op: "Identity"
  input: "save/RestoreV2:902"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_902"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_3/output/dense/kernel/adam_v"
  input: "save/Identity_903"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_904"
  op: "Identity"
  input: "save/RestoreV2:903"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_903"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta"
  input: "save/Identity_904"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_905"
  op: "Identity"
  input: "save/RestoreV2:904"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_904"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_905"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_906"
  op: "Identity"
  input: "save/RestoreV2:905"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_905"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_906"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_907"
  op: "Identity"
  input: "save/RestoreV2:906"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_906"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma"
  input: "save/Identity_907"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_908"
  op: "Identity"
  input: "save/RestoreV2:907"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_907"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_908"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_909"
  op: "Identity"
  input: "save/RestoreV2:908"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_908"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_909"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_910"
  op: "Identity"
  input: "save/RestoreV2:909"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_909"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias"
  input: "save/Identity_910"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_911"
  op: "Identity"
  input: "save/RestoreV2:910"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_910"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_m"
  input: "save/Identity_911"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_912"
  op: "Identity"
  input: "save/RestoreV2:911"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_911"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/bias/adam_v"
  input: "save/Identity_912"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_913"
  op: "Identity"
  input: "save/RestoreV2:912"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_912"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel"
  input: "save/Identity_913"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_914"
  op: "Identity"
  input: "save/RestoreV2:913"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_913"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_m"
  input: "save/Identity_914"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_915"
  op: "Identity"
  input: "save/RestoreV2:914"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_914"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/output/dense/kernel/adam_v"
  input: "save/Identity_915"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_916"
  op: "Identity"
  input: "save/RestoreV2:915"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_915"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias"
  input: "save/Identity_916"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_917"
  op: "Identity"
  input: "save/RestoreV2:916"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_916"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_m"
  input: "save/Identity_917"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_918"
  op: "Identity"
  input: "save/RestoreV2:917"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_917"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/bias/adam_v"
  input: "save/Identity_918"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_919"
  op: "Identity"
  input: "save/RestoreV2:918"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_918"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel"
  input: "save/Identity_919"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_920"
  op: "Identity"
  input: "save/RestoreV2:919"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_919"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_m"
  input: "save/Identity_920"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_921"
  op: "Identity"
  input: "save/RestoreV2:920"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_920"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/key/kernel/adam_v"
  input: "save/Identity_921"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_922"
  op: "Identity"
  input: "save/RestoreV2:921"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_921"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias"
  input: "save/Identity_922"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_923"
  op: "Identity"
  input: "save/RestoreV2:922"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_922"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_m"
  input: "save/Identity_923"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_924"
  op: "Identity"
  input: "save/RestoreV2:923"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_923"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/bias/adam_v"
  input: "save/Identity_924"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_925"
  op: "Identity"
  input: "save/RestoreV2:924"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_924"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel"
  input: "save/Identity_925"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_926"
  op: "Identity"
  input: "save/RestoreV2:925"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_925"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_m"
  input: "save/Identity_926"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_927"
  op: "Identity"
  input: "save/RestoreV2:926"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_926"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/query/kernel/adam_v"
  input: "save/Identity_927"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_928"
  op: "Identity"
  input: "save/RestoreV2:927"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_927"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias"
  input: "save/Identity_928"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_929"
  op: "Identity"
  input: "save/RestoreV2:928"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_928"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_m"
  input: "save/Identity_929"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_930"
  op: "Identity"
  input: "save/RestoreV2:929"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_929"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/bias/adam_v"
  input: "save/Identity_930"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_931"
  op: "Identity"
  input: "save/RestoreV2:930"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_930"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel"
  input: "save/Identity_931"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_932"
  op: "Identity"
  input: "save/RestoreV2:931"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_931"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_m"
  input: "save/Identity_932"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_933"
  op: "Identity"
  input: "save/RestoreV2:932"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_932"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/attention/self/value/kernel/adam_v"
  input: "save/Identity_933"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_934"
  op: "Identity"
  input: "save/RestoreV2:933"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_933"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias"
  input: "save/Identity_934"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_935"
  op: "Identity"
  input: "save/RestoreV2:934"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_934"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_m"
  input: "save/Identity_935"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_936"
  op: "Identity"
  input: "save/RestoreV2:935"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_935"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/bias/adam_v"
  input: "save/Identity_936"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_937"
  op: "Identity"
  input: "save/RestoreV2:936"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_936"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel"
  input: "save/Identity_937"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_938"
  op: "Identity"
  input: "save/RestoreV2:937"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_937"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_m"
  input: "save/Identity_938"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_939"
  op: "Identity"
  input: "save/RestoreV2:938"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_938"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/intermediate/dense/kernel/adam_v"
  input: "save/Identity_939"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_940"
  op: "Identity"
  input: "save/RestoreV2:939"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_939"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta"
  input: "save/Identity_940"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_941"
  op: "Identity"
  input: "save/RestoreV2:940"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_940"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_m"
  input: "save/Identity_941"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_942"
  op: "Identity"
  input: "save/RestoreV2:941"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_941"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/beta/adam_v"
  input: "save/Identity_942"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_943"
  op: "Identity"
  input: "save/RestoreV2:942"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_942"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma"
  input: "save/Identity_943"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_944"
  op: "Identity"
  input: "save/RestoreV2:943"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_943"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_944"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_945"
  op: "Identity"
  input: "save/RestoreV2:944"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_944"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_945"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_946"
  op: "Identity"
  input: "save/RestoreV2:945"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_945"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias"
  input: "save/Identity_946"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_947"
  op: "Identity"
  input: "save/RestoreV2:946"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_946"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_m"
  input: "save/Identity_947"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_948"
  op: "Identity"
  input: "save/RestoreV2:947"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_947"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/bias/adam_v"
  input: "save/Identity_948"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_949"
  op: "Identity"
  input: "save/RestoreV2:948"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_948"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel"
  input: "save/Identity_949"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_950"
  op: "Identity"
  input: "save/RestoreV2:949"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_949"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_m"
  input: "save/Identity_950"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_951"
  op: "Identity"
  input: "save/RestoreV2:950"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_950"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_4/output/dense/kernel/adam_v"
  input: "save/Identity_951"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_952"
  op: "Identity"
  input: "save/RestoreV2:951"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_951"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta"
  input: "save/Identity_952"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_953"
  op: "Identity"
  input: "save/RestoreV2:952"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_952"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_953"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_954"
  op: "Identity"
  input: "save/RestoreV2:953"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_953"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_954"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_955"
  op: "Identity"
  input: "save/RestoreV2:954"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_954"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma"
  input: "save/Identity_955"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_956"
  op: "Identity"
  input: "save/RestoreV2:955"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_955"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_956"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_957"
  op: "Identity"
  input: "save/RestoreV2:956"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_956"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_957"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_958"
  op: "Identity"
  input: "save/RestoreV2:957"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_957"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias"
  input: "save/Identity_958"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_959"
  op: "Identity"
  input: "save/RestoreV2:958"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_958"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_m"
  input: "save/Identity_959"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_960"
  op: "Identity"
  input: "save/RestoreV2:959"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_959"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/bias/adam_v"
  input: "save/Identity_960"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_961"
  op: "Identity"
  input: "save/RestoreV2:960"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_960"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel"
  input: "save/Identity_961"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_962"
  op: "Identity"
  input: "save/RestoreV2:961"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_961"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_m"
  input: "save/Identity_962"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_963"
  op: "Identity"
  input: "save/RestoreV2:962"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_962"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/output/dense/kernel/adam_v"
  input: "save/Identity_963"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_964"
  op: "Identity"
  input: "save/RestoreV2:963"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_963"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias"
  input: "save/Identity_964"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_965"
  op: "Identity"
  input: "save/RestoreV2:964"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_964"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_m"
  input: "save/Identity_965"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_966"
  op: "Identity"
  input: "save/RestoreV2:965"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_965"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/bias/adam_v"
  input: "save/Identity_966"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_967"
  op: "Identity"
  input: "save/RestoreV2:966"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_966"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel"
  input: "save/Identity_967"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_968"
  op: "Identity"
  input: "save/RestoreV2:967"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_967"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_m"
  input: "save/Identity_968"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_969"
  op: "Identity"
  input: "save/RestoreV2:968"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_968"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/key/kernel/adam_v"
  input: "save/Identity_969"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_970"
  op: "Identity"
  input: "save/RestoreV2:969"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_969"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias"
  input: "save/Identity_970"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_971"
  op: "Identity"
  input: "save/RestoreV2:970"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_970"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_m"
  input: "save/Identity_971"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_972"
  op: "Identity"
  input: "save/RestoreV2:971"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_971"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/bias/adam_v"
  input: "save/Identity_972"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_973"
  op: "Identity"
  input: "save/RestoreV2:972"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_972"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel"
  input: "save/Identity_973"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_974"
  op: "Identity"
  input: "save/RestoreV2:973"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_973"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_m"
  input: "save/Identity_974"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_975"
  op: "Identity"
  input: "save/RestoreV2:974"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_974"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/query/kernel/adam_v"
  input: "save/Identity_975"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_976"
  op: "Identity"
  input: "save/RestoreV2:975"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_975"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias"
  input: "save/Identity_976"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_977"
  op: "Identity"
  input: "save/RestoreV2:976"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_976"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_m"
  input: "save/Identity_977"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_978"
  op: "Identity"
  input: "save/RestoreV2:977"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_977"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/bias/adam_v"
  input: "save/Identity_978"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_979"
  op: "Identity"
  input: "save/RestoreV2:978"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_978"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel"
  input: "save/Identity_979"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_980"
  op: "Identity"
  input: "save/RestoreV2:979"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_979"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_m"
  input: "save/Identity_980"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_981"
  op: "Identity"
  input: "save/RestoreV2:980"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_980"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/attention/self/value/kernel/adam_v"
  input: "save/Identity_981"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_982"
  op: "Identity"
  input: "save/RestoreV2:981"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_981"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias"
  input: "save/Identity_982"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_983"
  op: "Identity"
  input: "save/RestoreV2:982"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_982"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_m"
  input: "save/Identity_983"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_984"
  op: "Identity"
  input: "save/RestoreV2:983"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_983"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/bias/adam_v"
  input: "save/Identity_984"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_985"
  op: "Identity"
  input: "save/RestoreV2:984"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_984"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel"
  input: "save/Identity_985"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_986"
  op: "Identity"
  input: "save/RestoreV2:985"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_985"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_m"
  input: "save/Identity_986"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_987"
  op: "Identity"
  input: "save/RestoreV2:986"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_986"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/intermediate/dense/kernel/adam_v"
  input: "save/Identity_987"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_988"
  op: "Identity"
  input: "save/RestoreV2:987"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_987"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta"
  input: "save/Identity_988"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_989"
  op: "Identity"
  input: "save/RestoreV2:988"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_988"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_m"
  input: "save/Identity_989"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_990"
  op: "Identity"
  input: "save/RestoreV2:989"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_989"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/beta/adam_v"
  input: "save/Identity_990"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_991"
  op: "Identity"
  input: "save/RestoreV2:990"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_990"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma"
  input: "save/Identity_991"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_992"
  op: "Identity"
  input: "save/RestoreV2:991"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_991"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_992"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_993"
  op: "Identity"
  input: "save/RestoreV2:992"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_992"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_993"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_994"
  op: "Identity"
  input: "save/RestoreV2:993"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_993"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias"
  input: "save/Identity_994"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_995"
  op: "Identity"
  input: "save/RestoreV2:994"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_994"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_m"
  input: "save/Identity_995"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_996"
  op: "Identity"
  input: "save/RestoreV2:995"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_995"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/bias/adam_v"
  input: "save/Identity_996"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_997"
  op: "Identity"
  input: "save/RestoreV2:996"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_996"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel"
  input: "save/Identity_997"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_998"
  op: "Identity"
  input: "save/RestoreV2:997"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_997"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_m"
  input: "save/Identity_998"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_999"
  op: "Identity"
  input: "save/RestoreV2:998"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_998"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_5/output/dense/kernel/adam_v"
  input: "save/Identity_999"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1000"
  op: "Identity"
  input: "save/RestoreV2:999"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_999"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta"
  input: "save/Identity_1000"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1001"
  op: "Identity"
  input: "save/RestoreV2:1000"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1000"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_1001"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1002"
  op: "Identity"
  input: "save/RestoreV2:1001"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1001"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_1002"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1003"
  op: "Identity"
  input: "save/RestoreV2:1002"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1002"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma"
  input: "save/Identity_1003"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1004"
  op: "Identity"
  input: "save/RestoreV2:1003"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1003"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_1004"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1005"
  op: "Identity"
  input: "save/RestoreV2:1004"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1004"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_1005"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1006"
  op: "Identity"
  input: "save/RestoreV2:1005"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1005"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias"
  input: "save/Identity_1006"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1007"
  op: "Identity"
  input: "save/RestoreV2:1006"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1006"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_m"
  input: "save/Identity_1007"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1008"
  op: "Identity"
  input: "save/RestoreV2:1007"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1007"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/bias/adam_v"
  input: "save/Identity_1008"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1009"
  op: "Identity"
  input: "save/RestoreV2:1008"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1008"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel"
  input: "save/Identity_1009"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1010"
  op: "Identity"
  input: "save/RestoreV2:1009"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1009"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_m"
  input: "save/Identity_1010"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1011"
  op: "Identity"
  input: "save/RestoreV2:1010"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1010"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/output/dense/kernel/adam_v"
  input: "save/Identity_1011"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1012"
  op: "Identity"
  input: "save/RestoreV2:1011"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1011"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias"
  input: "save/Identity_1012"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1013"
  op: "Identity"
  input: "save/RestoreV2:1012"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1012"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_m"
  input: "save/Identity_1013"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1014"
  op: "Identity"
  input: "save/RestoreV2:1013"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1013"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/bias/adam_v"
  input: "save/Identity_1014"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1015"
  op: "Identity"
  input: "save/RestoreV2:1014"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1014"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel"
  input: "save/Identity_1015"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1016"
  op: "Identity"
  input: "save/RestoreV2:1015"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1015"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_m"
  input: "save/Identity_1016"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1017"
  op: "Identity"
  input: "save/RestoreV2:1016"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1016"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/key/kernel/adam_v"
  input: "save/Identity_1017"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1018"
  op: "Identity"
  input: "save/RestoreV2:1017"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1017"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias"
  input: "save/Identity_1018"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1019"
  op: "Identity"
  input: "save/RestoreV2:1018"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1018"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_m"
  input: "save/Identity_1019"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1020"
  op: "Identity"
  input: "save/RestoreV2:1019"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1019"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/bias/adam_v"
  input: "save/Identity_1020"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1021"
  op: "Identity"
  input: "save/RestoreV2:1020"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1020"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel"
  input: "save/Identity_1021"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1022"
  op: "Identity"
  input: "save/RestoreV2:1021"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1021"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_m"
  input: "save/Identity_1022"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1023"
  op: "Identity"
  input: "save/RestoreV2:1022"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1022"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/query/kernel/adam_v"
  input: "save/Identity_1023"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1024"
  op: "Identity"
  input: "save/RestoreV2:1023"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1023"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias"
  input: "save/Identity_1024"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1025"
  op: "Identity"
  input: "save/RestoreV2:1024"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1024"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_m"
  input: "save/Identity_1025"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1026"
  op: "Identity"
  input: "save/RestoreV2:1025"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1025"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/bias/adam_v"
  input: "save/Identity_1026"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1027"
  op: "Identity"
  input: "save/RestoreV2:1026"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1026"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel"
  input: "save/Identity_1027"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1028"
  op: "Identity"
  input: "save/RestoreV2:1027"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1027"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_m"
  input: "save/Identity_1028"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1029"
  op: "Identity"
  input: "save/RestoreV2:1028"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1028"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/attention/self/value/kernel/adam_v"
  input: "save/Identity_1029"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1030"
  op: "Identity"
  input: "save/RestoreV2:1029"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1029"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias"
  input: "save/Identity_1030"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1031"
  op: "Identity"
  input: "save/RestoreV2:1030"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1030"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_m"
  input: "save/Identity_1031"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1032"
  op: "Identity"
  input: "save/RestoreV2:1031"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1031"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/bias/adam_v"
  input: "save/Identity_1032"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1033"
  op: "Identity"
  input: "save/RestoreV2:1032"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1032"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel"
  input: "save/Identity_1033"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1034"
  op: "Identity"
  input: "save/RestoreV2:1033"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1033"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_m"
  input: "save/Identity_1034"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1035"
  op: "Identity"
  input: "save/RestoreV2:1034"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1034"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/intermediate/dense/kernel/adam_v"
  input: "save/Identity_1035"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1036"
  op: "Identity"
  input: "save/RestoreV2:1035"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1035"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta"
  input: "save/Identity_1036"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1037"
  op: "Identity"
  input: "save/RestoreV2:1036"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1036"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_m"
  input: "save/Identity_1037"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1038"
  op: "Identity"
  input: "save/RestoreV2:1037"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1037"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/beta/adam_v"
  input: "save/Identity_1038"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1039"
  op: "Identity"
  input: "save/RestoreV2:1038"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1038"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma"
  input: "save/Identity_1039"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1040"
  op: "Identity"
  input: "save/RestoreV2:1039"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1039"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_1040"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1041"
  op: "Identity"
  input: "save/RestoreV2:1040"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1040"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_1041"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1042"
  op: "Identity"
  input: "save/RestoreV2:1041"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1041"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias"
  input: "save/Identity_1042"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1043"
  op: "Identity"
  input: "save/RestoreV2:1042"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1042"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_m"
  input: "save/Identity_1043"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1044"
  op: "Identity"
  input: "save/RestoreV2:1043"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1043"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/bias/adam_v"
  input: "save/Identity_1044"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1045"
  op: "Identity"
  input: "save/RestoreV2:1044"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1044"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel"
  input: "save/Identity_1045"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1046"
  op: "Identity"
  input: "save/RestoreV2:1045"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1045"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_m"
  input: "save/Identity_1046"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1047"
  op: "Identity"
  input: "save/RestoreV2:1046"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1046"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_6/output/dense/kernel/adam_v"
  input: "save/Identity_1047"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1048"
  op: "Identity"
  input: "save/RestoreV2:1047"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1047"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta"
  input: "save/Identity_1048"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1049"
  op: "Identity"
  input: "save/RestoreV2:1048"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1048"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_1049"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1050"
  op: "Identity"
  input: "save/RestoreV2:1049"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1049"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_1050"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1051"
  op: "Identity"
  input: "save/RestoreV2:1050"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1050"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma"
  input: "save/Identity_1051"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1052"
  op: "Identity"
  input: "save/RestoreV2:1051"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1051"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_1052"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1053"
  op: "Identity"
  input: "save/RestoreV2:1052"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1052"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_1053"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1054"
  op: "Identity"
  input: "save/RestoreV2:1053"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1053"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias"
  input: "save/Identity_1054"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1055"
  op: "Identity"
  input: "save/RestoreV2:1054"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1054"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_m"
  input: "save/Identity_1055"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1056"
  op: "Identity"
  input: "save/RestoreV2:1055"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1055"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/bias/adam_v"
  input: "save/Identity_1056"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1057"
  op: "Identity"
  input: "save/RestoreV2:1056"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1056"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel"
  input: "save/Identity_1057"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1058"
  op: "Identity"
  input: "save/RestoreV2:1057"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1057"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_m"
  input: "save/Identity_1058"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1059"
  op: "Identity"
  input: "save/RestoreV2:1058"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1058"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/output/dense/kernel/adam_v"
  input: "save/Identity_1059"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1060"
  op: "Identity"
  input: "save/RestoreV2:1059"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1059"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias"
  input: "save/Identity_1060"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1061"
  op: "Identity"
  input: "save/RestoreV2:1060"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1060"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_m"
  input: "save/Identity_1061"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1062"
  op: "Identity"
  input: "save/RestoreV2:1061"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1061"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/bias/adam_v"
  input: "save/Identity_1062"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1063"
  op: "Identity"
  input: "save/RestoreV2:1062"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1062"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel"
  input: "save/Identity_1063"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1064"
  op: "Identity"
  input: "save/RestoreV2:1063"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1063"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_m"
  input: "save/Identity_1064"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1065"
  op: "Identity"
  input: "save/RestoreV2:1064"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1064"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/key/kernel/adam_v"
  input: "save/Identity_1065"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1066"
  op: "Identity"
  input: "save/RestoreV2:1065"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1065"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias"
  input: "save/Identity_1066"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1067"
  op: "Identity"
  input: "save/RestoreV2:1066"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1066"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_m"
  input: "save/Identity_1067"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1068"
  op: "Identity"
  input: "save/RestoreV2:1067"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1067"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/bias/adam_v"
  input: "save/Identity_1068"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1069"
  op: "Identity"
  input: "save/RestoreV2:1068"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1068"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel"
  input: "save/Identity_1069"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1070"
  op: "Identity"
  input: "save/RestoreV2:1069"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1069"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_m"
  input: "save/Identity_1070"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1071"
  op: "Identity"
  input: "save/RestoreV2:1070"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1070"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/query/kernel/adam_v"
  input: "save/Identity_1071"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1072"
  op: "Identity"
  input: "save/RestoreV2:1071"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1071"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias"
  input: "save/Identity_1072"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1073"
  op: "Identity"
  input: "save/RestoreV2:1072"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1072"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_m"
  input: "save/Identity_1073"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1074"
  op: "Identity"
  input: "save/RestoreV2:1073"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1073"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/bias/adam_v"
  input: "save/Identity_1074"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1075"
  op: "Identity"
  input: "save/RestoreV2:1074"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1074"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel"
  input: "save/Identity_1075"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1076"
  op: "Identity"
  input: "save/RestoreV2:1075"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1075"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_m"
  input: "save/Identity_1076"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1077"
  op: "Identity"
  input: "save/RestoreV2:1076"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1076"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/attention/self/value/kernel/adam_v"
  input: "save/Identity_1077"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1078"
  op: "Identity"
  input: "save/RestoreV2:1077"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1077"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias"
  input: "save/Identity_1078"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1079"
  op: "Identity"
  input: "save/RestoreV2:1078"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1078"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_m"
  input: "save/Identity_1079"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1080"
  op: "Identity"
  input: "save/RestoreV2:1079"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1079"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/bias/adam_v"
  input: "save/Identity_1080"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1081"
  op: "Identity"
  input: "save/RestoreV2:1080"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1080"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel"
  input: "save/Identity_1081"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1082"
  op: "Identity"
  input: "save/RestoreV2:1081"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1081"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_m"
  input: "save/Identity_1082"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1083"
  op: "Identity"
  input: "save/RestoreV2:1082"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1082"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/intermediate/dense/kernel/adam_v"
  input: "save/Identity_1083"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1084"
  op: "Identity"
  input: "save/RestoreV2:1083"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1083"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta"
  input: "save/Identity_1084"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1085"
  op: "Identity"
  input: "save/RestoreV2:1084"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1084"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_m"
  input: "save/Identity_1085"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1086"
  op: "Identity"
  input: "save/RestoreV2:1085"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1085"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/beta/adam_v"
  input: "save/Identity_1086"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1087"
  op: "Identity"
  input: "save/RestoreV2:1086"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1086"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma"
  input: "save/Identity_1087"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1088"
  op: "Identity"
  input: "save/RestoreV2:1087"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1087"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_1088"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1089"
  op: "Identity"
  input: "save/RestoreV2:1088"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1088"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_1089"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1090"
  op: "Identity"
  input: "save/RestoreV2:1089"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1089"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias"
  input: "save/Identity_1090"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1091"
  op: "Identity"
  input: "save/RestoreV2:1090"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1090"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_m"
  input: "save/Identity_1091"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1092"
  op: "Identity"
  input: "save/RestoreV2:1091"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1091"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/bias/adam_v"
  input: "save/Identity_1092"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1093"
  op: "Identity"
  input: "save/RestoreV2:1092"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1092"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel"
  input: "save/Identity_1093"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1094"
  op: "Identity"
  input: "save/RestoreV2:1093"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1093"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_m"
  input: "save/Identity_1094"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1095"
  op: "Identity"
  input: "save/RestoreV2:1094"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1094"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_7/output/dense/kernel/adam_v"
  input: "save/Identity_1095"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1096"
  op: "Identity"
  input: "save/RestoreV2:1095"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1095"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta"
  input: "save/Identity_1096"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1097"
  op: "Identity"
  input: "save/RestoreV2:1096"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1096"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_1097"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1098"
  op: "Identity"
  input: "save/RestoreV2:1097"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1097"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_1098"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1099"
  op: "Identity"
  input: "save/RestoreV2:1098"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1098"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma"
  input: "save/Identity_1099"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1100"
  op: "Identity"
  input: "save/RestoreV2:1099"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1099"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_1100"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1101"
  op: "Identity"
  input: "save/RestoreV2:1100"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1100"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_1101"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1102"
  op: "Identity"
  input: "save/RestoreV2:1101"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1101"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias"
  input: "save/Identity_1102"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1103"
  op: "Identity"
  input: "save/RestoreV2:1102"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1102"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_m"
  input: "save/Identity_1103"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1104"
  op: "Identity"
  input: "save/RestoreV2:1103"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1103"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/bias/adam_v"
  input: "save/Identity_1104"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1105"
  op: "Identity"
  input: "save/RestoreV2:1104"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1104"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel"
  input: "save/Identity_1105"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1106"
  op: "Identity"
  input: "save/RestoreV2:1105"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1105"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_m"
  input: "save/Identity_1106"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1107"
  op: "Identity"
  input: "save/RestoreV2:1106"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1106"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/output/dense/kernel/adam_v"
  input: "save/Identity_1107"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1108"
  op: "Identity"
  input: "save/RestoreV2:1107"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1107"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias"
  input: "save/Identity_1108"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1109"
  op: "Identity"
  input: "save/RestoreV2:1108"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1108"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_m"
  input: "save/Identity_1109"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1110"
  op: "Identity"
  input: "save/RestoreV2:1109"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1109"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/bias/adam_v"
  input: "save/Identity_1110"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1111"
  op: "Identity"
  input: "save/RestoreV2:1110"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1110"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel"
  input: "save/Identity_1111"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1112"
  op: "Identity"
  input: "save/RestoreV2:1111"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1111"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_m"
  input: "save/Identity_1112"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1113"
  op: "Identity"
  input: "save/RestoreV2:1112"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1112"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/key/kernel/adam_v"
  input: "save/Identity_1113"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1114"
  op: "Identity"
  input: "save/RestoreV2:1113"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1113"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias"
  input: "save/Identity_1114"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1115"
  op: "Identity"
  input: "save/RestoreV2:1114"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1114"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_m"
  input: "save/Identity_1115"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1116"
  op: "Identity"
  input: "save/RestoreV2:1115"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1115"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/bias/adam_v"
  input: "save/Identity_1116"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1117"
  op: "Identity"
  input: "save/RestoreV2:1116"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1116"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel"
  input: "save/Identity_1117"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1118"
  op: "Identity"
  input: "save/RestoreV2:1117"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1117"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_m"
  input: "save/Identity_1118"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1119"
  op: "Identity"
  input: "save/RestoreV2:1118"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1118"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/query/kernel/adam_v"
  input: "save/Identity_1119"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1120"
  op: "Identity"
  input: "save/RestoreV2:1119"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1119"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias"
  input: "save/Identity_1120"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1121"
  op: "Identity"
  input: "save/RestoreV2:1120"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1120"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_m"
  input: "save/Identity_1121"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1122"
  op: "Identity"
  input: "save/RestoreV2:1121"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1121"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/bias/adam_v"
  input: "save/Identity_1122"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1123"
  op: "Identity"
  input: "save/RestoreV2:1122"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1122"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel"
  input: "save/Identity_1123"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1124"
  op: "Identity"
  input: "save/RestoreV2:1123"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1123"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_m"
  input: "save/Identity_1124"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1125"
  op: "Identity"
  input: "save/RestoreV2:1124"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1124"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/attention/self/value/kernel/adam_v"
  input: "save/Identity_1125"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1126"
  op: "Identity"
  input: "save/RestoreV2:1125"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1125"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias"
  input: "save/Identity_1126"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1127"
  op: "Identity"
  input: "save/RestoreV2:1126"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1126"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_m"
  input: "save/Identity_1127"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1128"
  op: "Identity"
  input: "save/RestoreV2:1127"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1127"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/bias/adam_v"
  input: "save/Identity_1128"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1129"
  op: "Identity"
  input: "save/RestoreV2:1128"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1128"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel"
  input: "save/Identity_1129"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1130"
  op: "Identity"
  input: "save/RestoreV2:1129"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1129"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_m"
  input: "save/Identity_1130"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1131"
  op: "Identity"
  input: "save/RestoreV2:1130"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1130"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/intermediate/dense/kernel/adam_v"
  input: "save/Identity_1131"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1132"
  op: "Identity"
  input: "save/RestoreV2:1131"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1131"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta"
  input: "save/Identity_1132"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1133"
  op: "Identity"
  input: "save/RestoreV2:1132"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1132"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_m"
  input: "save/Identity_1133"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1134"
  op: "Identity"
  input: "save/RestoreV2:1133"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1133"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/beta/adam_v"
  input: "save/Identity_1134"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1135"
  op: "Identity"
  input: "save/RestoreV2:1134"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1134"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma"
  input: "save/Identity_1135"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1136"
  op: "Identity"
  input: "save/RestoreV2:1135"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1135"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_1136"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1137"
  op: "Identity"
  input: "save/RestoreV2:1136"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1136"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_1137"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1138"
  op: "Identity"
  input: "save/RestoreV2:1137"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1137"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias"
  input: "save/Identity_1138"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1139"
  op: "Identity"
  input: "save/RestoreV2:1138"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1138"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_m"
  input: "save/Identity_1139"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1140"
  op: "Identity"
  input: "save/RestoreV2:1139"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1139"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/bias/adam_v"
  input: "save/Identity_1140"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1141"
  op: "Identity"
  input: "save/RestoreV2:1140"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1140"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel"
  input: "save/Identity_1141"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1142"
  op: "Identity"
  input: "save/RestoreV2:1141"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1141"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_m"
  input: "save/Identity_1142"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1143"
  op: "Identity"
  input: "save/RestoreV2:1142"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1142"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_8/output/dense/kernel/adam_v"
  input: "save/Identity_1143"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1144"
  op: "Identity"
  input: "save/RestoreV2:1143"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1143"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta"
  input: "save/Identity_1144"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1145"
  op: "Identity"
  input: "save/RestoreV2:1144"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1144"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_m"
  input: "save/Identity_1145"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1146"
  op: "Identity"
  input: "save/RestoreV2:1145"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1145"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/beta/adam_v"
  input: "save/Identity_1146"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1147"
  op: "Identity"
  input: "save/RestoreV2:1146"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1146"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma"
  input: "save/Identity_1147"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1148"
  op: "Identity"
  input: "save/RestoreV2:1147"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1147"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_1148"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1149"
  op: "Identity"
  input: "save/RestoreV2:1148"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1148"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_1149"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1150"
  op: "Identity"
  input: "save/RestoreV2:1149"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1149"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias"
  input: "save/Identity_1150"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1151"
  op: "Identity"
  input: "save/RestoreV2:1150"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1150"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_m"
  input: "save/Identity_1151"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1152"
  op: "Identity"
  input: "save/RestoreV2:1151"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1151"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/bias/adam_v"
  input: "save/Identity_1152"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1153"
  op: "Identity"
  input: "save/RestoreV2:1152"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1152"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel"
  input: "save/Identity_1153"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1154"
  op: "Identity"
  input: "save/RestoreV2:1153"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1153"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_m"
  input: "save/Identity_1154"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1155"
  op: "Identity"
  input: "save/RestoreV2:1154"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1154"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/output/dense/kernel/adam_v"
  input: "save/Identity_1155"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1156"
  op: "Identity"
  input: "save/RestoreV2:1155"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1155"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias"
  input: "save/Identity_1156"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1157"
  op: "Identity"
  input: "save/RestoreV2:1156"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1156"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_m"
  input: "save/Identity_1157"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1158"
  op: "Identity"
  input: "save/RestoreV2:1157"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1157"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/bias/adam_v"
  input: "save/Identity_1158"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1159"
  op: "Identity"
  input: "save/RestoreV2:1158"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1158"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel"
  input: "save/Identity_1159"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1160"
  op: "Identity"
  input: "save/RestoreV2:1159"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1159"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_m"
  input: "save/Identity_1160"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1161"
  op: "Identity"
  input: "save/RestoreV2:1160"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1160"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/key/kernel/adam_v"
  input: "save/Identity_1161"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1162"
  op: "Identity"
  input: "save/RestoreV2:1161"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1161"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias"
  input: "save/Identity_1162"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1163"
  op: "Identity"
  input: "save/RestoreV2:1162"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1162"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_m"
  input: "save/Identity_1163"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1164"
  op: "Identity"
  input: "save/RestoreV2:1163"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1163"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/bias/adam_v"
  input: "save/Identity_1164"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1165"
  op: "Identity"
  input: "save/RestoreV2:1164"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1164"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel"
  input: "save/Identity_1165"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1166"
  op: "Identity"
  input: "save/RestoreV2:1165"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1165"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_m"
  input: "save/Identity_1166"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1167"
  op: "Identity"
  input: "save/RestoreV2:1166"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1166"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/query/kernel/adam_v"
  input: "save/Identity_1167"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1168"
  op: "Identity"
  input: "save/RestoreV2:1167"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1167"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias"
  input: "save/Identity_1168"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1169"
  op: "Identity"
  input: "save/RestoreV2:1168"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1168"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_m"
  input: "save/Identity_1169"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1170"
  op: "Identity"
  input: "save/RestoreV2:1169"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1169"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/bias/adam_v"
  input: "save/Identity_1170"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1171"
  op: "Identity"
  input: "save/RestoreV2:1170"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1170"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel"
  input: "save/Identity_1171"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1172"
  op: "Identity"
  input: "save/RestoreV2:1171"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1171"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_m"
  input: "save/Identity_1172"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1173"
  op: "Identity"
  input: "save/RestoreV2:1172"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1172"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/attention/self/value/kernel/adam_v"
  input: "save/Identity_1173"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1174"
  op: "Identity"
  input: "save/RestoreV2:1173"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1173"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias"
  input: "save/Identity_1174"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1175"
  op: "Identity"
  input: "save/RestoreV2:1174"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1174"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_m"
  input: "save/Identity_1175"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1176"
  op: "Identity"
  input: "save/RestoreV2:1175"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1175"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/bias/adam_v"
  input: "save/Identity_1176"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1177"
  op: "Identity"
  input: "save/RestoreV2:1176"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1176"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel"
  input: "save/Identity_1177"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1178"
  op: "Identity"
  input: "save/RestoreV2:1177"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1177"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_m"
  input: "save/Identity_1178"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1179"
  op: "Identity"
  input: "save/RestoreV2:1178"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1178"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/intermediate/dense/kernel/adam_v"
  input: "save/Identity_1179"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1180"
  op: "Identity"
  input: "save/RestoreV2:1179"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1179"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta"
  input: "save/Identity_1180"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1181"
  op: "Identity"
  input: "save/RestoreV2:1180"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1180"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_m"
  input: "save/Identity_1181"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1182"
  op: "Identity"
  input: "save/RestoreV2:1181"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1181"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/beta/adam_v"
  input: "save/Identity_1182"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1183"
  op: "Identity"
  input: "save/RestoreV2:1182"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1182"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma"
  input: "save/Identity_1183"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1184"
  op: "Identity"
  input: "save/RestoreV2:1183"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1183"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_m"
  input: "save/Identity_1184"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1185"
  op: "Identity"
  input: "save/RestoreV2:1184"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1184"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/LayerNorm/gamma/adam_v"
  input: "save/Identity_1185"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1186"
  op: "Identity"
  input: "save/RestoreV2:1185"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1185"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias"
  input: "save/Identity_1186"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1187"
  op: "Identity"
  input: "save/RestoreV2:1186"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1186"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_m"
  input: "save/Identity_1187"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1188"
  op: "Identity"
  input: "save/RestoreV2:1187"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1187"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/bias/adam_v"
  input: "save/Identity_1188"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1189"
  op: "Identity"
  input: "save/RestoreV2:1188"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1188"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel"
  input: "save/Identity_1189"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1190"
  op: "Identity"
  input: "save/RestoreV2:1189"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1189"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_m"
  input: "save/Identity_1190"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1191"
  op: "Identity"
  input: "save/RestoreV2:1190"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1190"
  op: "AssignVariableOp"
  input: "generator/encoder/layer_9/output/dense/kernel/adam_v"
  input: "save/Identity_1191"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1192"
  op: "Identity"
  input: "save/RestoreV2:1191"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1191"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/beta"
  input: "save/Identity_1192"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1193"
  op: "Identity"
  input: "save/RestoreV2:1192"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1192"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/beta/adam_m"
  input: "save/Identity_1193"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1194"
  op: "Identity"
  input: "save/RestoreV2:1193"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1193"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/beta/adam_v"
  input: "save/Identity_1194"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1195"
  op: "Identity"
  input: "save/RestoreV2:1194"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1194"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/gamma"
  input: "save/Identity_1195"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1196"
  op: "Identity"
  input: "save/RestoreV2:1195"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1195"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/gamma/adam_m"
  input: "save/Identity_1196"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1197"
  op: "Identity"
  input: "save/RestoreV2:1196"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1196"
  op: "AssignVariableOp"
  input: "generator_predictions/LayerNorm/gamma/adam_v"
  input: "save/Identity_1197"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1198"
  op: "Identity"
  input: "save/RestoreV2:1197"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1197"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/bias"
  input: "save/Identity_1198"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1199"
  op: "Identity"
  input: "save/RestoreV2:1198"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1198"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/bias/adam_m"
  input: "save/Identity_1199"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1200"
  op: "Identity"
  input: "save/RestoreV2:1199"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1199"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/bias/adam_v"
  input: "save/Identity_1200"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1201"
  op: "Identity"
  input: "save/RestoreV2:1200"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1200"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/kernel"
  input: "save/Identity_1201"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1202"
  op: "Identity"
  input: "save/RestoreV2:1201"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1201"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/kernel/adam_m"
  input: "save/Identity_1202"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1203"
  op: "Identity"
  input: "save/RestoreV2:1202"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1202"
  op: "AssignVariableOp"
  input: "generator_predictions/dense/kernel/adam_v"
  input: "save/Identity_1203"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1204"
  op: "Identity"
  input: "save/RestoreV2:1203"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1203"
  op: "AssignVariableOp"
  input: "generator_predictions/output_bias"
  input: "save/Identity_1204"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1205"
  op: "Identity"
  input: "save/RestoreV2:1204"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1204"
  op: "AssignVariableOp"
  input: "generator_predictions/output_bias/adam_m"
  input: "save/Identity_1205"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1206"
  op: "Identity"
  input: "save/RestoreV2:1205"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1205"
  op: "AssignVariableOp"
  input: "generator_predictions/output_bias/adam_v"
  input: "save/Identity_1206"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "save/Identity_1207"
  op: "Identity"
  input: "save/RestoreV2:1206"
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          unknown_rank: true
        }
      }
    }
  }
}
node {
  name: "save/AssignVariableOp_1206"
  op: "AssignVariableOp"
  input: "global_step"
  input: "save/Identity_1207"
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "save/restore_shard"
  op: "NoOp"
  input: "^save/AssignVariableOp"
  input: "^save/AssignVariableOp_1"
  input: "^save/AssignVariableOp_10"
  input: "^save/AssignVariableOp_100"
  input: "^save/AssignVariableOp_1000"
  input: "^save/AssignVariableOp_1001"
  input: "^save/AssignVariableOp_1002"
  input: "^save/AssignVariableOp_1003"
  input: "^save/AssignVariableOp_1004"
  input: "^save/AssignVariableOp_1005"
  input: "^save/AssignVariableOp_1006"
  input: "^save/AssignVariableOp_1007"
  input: "^save/AssignVariableOp_1008"
  input: "^save/AssignVariableOp_1009"
  input: "^save/AssignVariableOp_101"
  input: "^save/AssignVariableOp_1010"
  input: "^save/AssignVariableOp_1011"
  input: "^save/AssignVariableOp_1012"
  input: "^save/AssignVariableOp_1013"
  input: "^save/AssignVariableOp_1014"
  input: "^save/AssignVariableOp_1015"
  input: "^save/AssignVariableOp_1016"
  input: "^save/AssignVariableOp_1017"
  input: "^save/AssignVariableOp_1018"
  input: "^save/AssignVariableOp_1019"
  input: "^save/AssignVariableOp_102"
  input: "^save/AssignVariableOp_1020"
  input: "^save/AssignVariableOp_1021"
  input: "^save/AssignVariableOp_1022"
  input: "^save/AssignVariableOp_1023"
  input: "^save/AssignVariableOp_1024"
  input: "^save/AssignVariableOp_1025"
  input: "^save/AssignVariableOp_1026"
  input: "^save/AssignVariableOp_1027"
  input: "^save/AssignVariableOp_1028"
  input: "^save/AssignVariableOp_1029"
  input: "^save/AssignVariableOp_103"
  input: "^save/AssignVariableOp_1030"
  input: "^save/AssignVariableOp_1031"
  input: "^save/AssignVariableOp_1032"
  input: "^save/AssignVariableOp_1033"
  input: "^save/AssignVariableOp_1034"
  input: "^save/AssignVariableOp_1035"
  input: "^save/AssignVariableOp_1036"
  input: "^save/AssignVariableOp_1037"
  input: "^save/AssignVariableOp_1038"
  input: "^save/AssignVariableOp_1039"
  input: "^save/AssignVariableOp_104"
  input: "^save/AssignVariableOp_1040"
  input: "^save/AssignVariableOp_1041"
  input: "^save/AssignVariableOp_1042"
  input: "^save/AssignVariableOp_1043"
  input: "^save/AssignVariableOp_1044"
  input: "^save/AssignVariableOp_1045"
  input: "^save/AssignVariableOp_1046"
  input: "^save/AssignVariableOp_1047"
  input: "^save/AssignVariableOp_1048"
  input: "^save/AssignVariableOp_1049"
  input: "^save/AssignVariableOp_105"
  input: "^save/AssignVariableOp_1050"
  input: "^save/AssignVariableOp_1051"
  input: "^save/AssignVariableOp_1052"
  input: "^save/AssignVariableOp_1053"
  input: "^save/AssignVariableOp_1054"
  input: "^save/AssignVariableOp_1055"
  input: "^save/AssignVariableOp_1056"
  input: "^save/AssignVariableOp_1057"
  input: "^save/AssignVariableOp_1058"
  input: "^save/AssignVariableOp_1059"
  input: "^save/AssignVariableOp_106"
  input: "^save/AssignVariableOp_1060"
  input: "^save/AssignVariableOp_1061"
  input: "^save/AssignVariableOp_1062"
  input: "^save/AssignVariableOp_1063"
  input: "^save/AssignVariableOp_1064"
  input: "^save/AssignVariableOp_1065"
  input: "^save/AssignVariableOp_1066"
  input: "^save/AssignVariableOp_1067"
  input: "^save/AssignVariableOp_1068"
  input: "^save/AssignVariableOp_1069"
  input: "^save/AssignVariableOp_107"
  input: "^save/AssignVariableOp_1070"
  input: "^save/AssignVariableOp_1071"
  input: "^save/AssignVariableOp_1072"
  input: "^save/AssignVariableOp_1073"
  input: "^save/AssignVariableOp_1074"
  input: "^save/AssignVariableOp_1075"
  input: "^save/AssignVariableOp_1076"
  input: "^save/AssignVariableOp_1077"
  input: "^save/AssignVariableOp_1078"
  input: "^save/AssignVariableOp_1079"
  input: "^save/AssignVariableOp_108"
  input: "^save/AssignVariableOp_1080"
  input: "^save/AssignVariableOp_1081"
  input: "^save/AssignVariableOp_1082"
  input: "^save/AssignVariableOp_1083"
  input: "^save/AssignVariableOp_1084"
  input: "^save/AssignVariableOp_1085"
  input: "^save/AssignVariableOp_1086"
  input: "^save/AssignVariableOp_1087"
  input: "^save/AssignVariableOp_1088"
  input: "^save/AssignVariableOp_1089"
  input: "^save/AssignVariableOp_109"
  input: "^save/AssignVariableOp_1090"
  input: "^save/AssignVariableOp_1091"
  input: "^save/AssignVariableOp_1092"
  input: "^save/AssignVariableOp_1093"
  input: "^save/AssignVariableOp_1094"
  input: "^save/AssignVariableOp_1095"
  input: "^save/AssignVariableOp_1096"
  input: "^save/AssignVariableOp_1097"
  input: "^save/AssignVariableOp_1098"
  input: "^save/AssignVariableOp_1099"
  input: "^save/AssignVariableOp_11"
  input: "^save/AssignVariableOp_110"
  input: "^save/AssignVariableOp_1100"
  input: "^save/AssignVariableOp_1101"
  input: "^save/AssignVariableOp_1102"
  input: "^save/AssignVariableOp_1103"
  input: "^save/AssignVariableOp_1104"
  input: "^save/AssignVariableOp_1105"
  input: "^save/AssignVariableOp_1106"
  input: "^save/AssignVariableOp_1107"
  input: "^save/AssignVariableOp_1108"
  input: "^save/AssignVariableOp_1109"
  input: "^save/AssignVariableOp_111"
  input: "^save/AssignVariableOp_1110"
  input: "^save/AssignVariableOp_1111"
  input: "^save/AssignVariableOp_1112"
  input: "^save/AssignVariableOp_1113"
  input: "^save/AssignVariableOp_1114"
  input: "^save/AssignVariableOp_1115"
  input: "^save/AssignVariableOp_1116"
  input: "^save/AssignVariableOp_1117"
  input: "^save/AssignVariableOp_1118"
  input: "^save/AssignVariableOp_1119"
  input: "^save/AssignVariableOp_112"
  input: "^save/AssignVariableOp_1120"
  input: "^save/AssignVariableOp_1121"
  input: "^save/AssignVariableOp_1122"
  input: "^save/AssignVariableOp_1123"
  input: "^save/AssignVariableOp_1124"
  input: "^save/AssignVariableOp_1125"
  input: "^save/AssignVariableOp_1126"
  input: "^save/AssignVariableOp_1127"
  input: "^save/AssignVariableOp_1128"
  input: "^save/AssignVariableOp_1129"
  input: "^save/AssignVariableOp_113"
  input: "^save/AssignVariableOp_1130"
  input: "^save/AssignVariableOp_1131"
  input: "^save/AssignVariableOp_1132"
  input: "^save/AssignVariableOp_1133"
  input: "^save/AssignVariableOp_1134"
  input: "^save/AssignVariableOp_1135"
  input: "^save/AssignVariableOp_1136"
  input: "^save/AssignVariableOp_1137"
  input: "^save/AssignVariableOp_1138"
  input: "^save/AssignVariableOp_1139"
  input: "^save/AssignVariableOp_114"
  input: "^save/AssignVariableOp_1140"
  input: "^save/AssignVariableOp_1141"
  input: "^save/AssignVariableOp_1142"
  input: "^save/AssignVariableOp_1143"
  input: "^save/AssignVariableOp_1144"
  input: "^save/AssignVariableOp_1145"
  input: "^save/AssignVariableOp_1146"
  input: "^save/AssignVariableOp_1147"
  input: "^save/AssignVariableOp_1148"
  input: "^save/AssignVariableOp_1149"
  input: "^save/AssignVariableOp_115"
  input: "^save/AssignVariableOp_1150"
  input: "^save/AssignVariableOp_1151"
  input: "^save/AssignVariableOp_1152"
  input: "^save/AssignVariableOp_1153"
  input: "^save/AssignVariableOp_1154"
  input: "^save/AssignVariableOp_1155"
  input: "^save/AssignVariableOp_1156"
  input: "^save/AssignVariableOp_1157"
  input: "^save/AssignVariableOp_1158"
  input: "^save/AssignVariableOp_1159"
  input: "^save/AssignVariableOp_116"
  input: "^save/AssignVariableOp_1160"
  input: "^save/AssignVariableOp_1161"
  input: "^save/AssignVariableOp_1162"
  input: "^save/AssignVariableOp_1163"
  input: "^save/AssignVariableOp_1164"
  input: "^save/AssignVariableOp_1165"
  input: "^save/AssignVariableOp_1166"
  input: "^save/AssignVariableOp_1167"
  input: "^save/AssignVariableOp_1168"
  input: "^save/AssignVariableOp_1169"
  input: "^save/AssignVariableOp_117"
  input: "^save/AssignVariableOp_1170"
  input: "^save/AssignVariableOp_1171"
  input: "^save/AssignVariableOp_1172"
  input: "^save/AssignVariableOp_1173"
  input: "^save/AssignVariableOp_1174"
  input: "^save/AssignVariableOp_1175"
  input: "^save/AssignVariableOp_1176"
  input: "^save/AssignVariableOp_1177"
  input: "^save/AssignVariableOp_1178"
  input: "^save/AssignVariableOp_1179"
  input: "^save/AssignVariableOp_118"
  input: "^save/AssignVariableOp_1180"
  input: "^save/AssignVariableOp_1181"
  input: "^save/AssignVariableOp_1182"
  input: "^save/AssignVariableOp_1183"
  input: "^save/AssignVariableOp_1184"
  input: "^save/AssignVariableOp_1185"
  input: "^save/AssignVariableOp_1186"
  input: "^save/AssignVariableOp_1187"
  input: "^save/AssignVariableOp_1188"
  input: "^save/AssignVariableOp_1189"
  input: "^save/AssignVariableOp_119"
  input: "^save/AssignVariableOp_1190"
  input: "^save/AssignVariableOp_1191"
  input: "^save/AssignVariableOp_1192"
  input: "^save/AssignVariableOp_1193"
  input: "^save/AssignVariableOp_1194"
  input: "^save/AssignVariableOp_1195"
  input: "^save/AssignVariableOp_1196"
  input: "^save/AssignVariableOp_1197"
  input: "^save/AssignVariableOp_1198"
  input: "^save/AssignVariableOp_1199"
  input: "^save/AssignVariableOp_12"
  input: "^save/AssignVariableOp_120"
  input: "^save/AssignVariableOp_1200"
  input: "^save/AssignVariableOp_1201"
  input: "^save/AssignVariableOp_1202"
  input: "^save/AssignVariableOp_1203"
  input: "^save/AssignVariableOp_1204"
  input: "^save/AssignVariableOp_1205"
  input: "^save/AssignVariableOp_1206"
  input: "^save/AssignVariableOp_121"
  input: "^save/AssignVariableOp_122"
  input: "^save/AssignVariableOp_123"
  input: "^save/AssignVariableOp_124"
  input: "^save/AssignVariableOp_125"
  input: "^save/AssignVariableOp_126"
  input: "^save/AssignVariableOp_127"
  input: "^save/AssignVariableOp_128"
  input: "^save/AssignVariableOp_129"
  input: "^save/AssignVariableOp_13"
  input: "^save/AssignVariableOp_130"
  input: "^save/AssignVariableOp_131"
  input: "^save/AssignVariableOp_132"
  input: "^save/AssignVariableOp_133"
  input: "^save/AssignVariableOp_134"
  input: "^save/AssignVariableOp_135"
  input: "^save/AssignVariableOp_136"
  input: "^save/AssignVariableOp_137"
  input: "^save/AssignVariableOp_138"
  input: "^save/AssignVariableOp_139"
  input: "^save/AssignVariableOp_14"
  input: "^save/AssignVariableOp_140"
  input: "^save/AssignVariableOp_141"
  input: "^save/AssignVariableOp_142"
  input: "^save/AssignVariableOp_143"
  input: "^save/AssignVariableOp_144"
  input: "^save/AssignVariableOp_145"
  input: "^save/AssignVariableOp_146"
  input: "^save/AssignVariableOp_147"
  input: "^save/AssignVariableOp_148"
  input: "^save/AssignVariableOp_149"
  input: "^save/AssignVariableOp_15"
  input: "^save/AssignVariableOp_150"
  input: "^save/AssignVariableOp_151"
  input: "^save/AssignVariableOp_152"
  input: "^save/AssignVariableOp_153"
  input: "^save/AssignVariableOp_154"
  input: "^save/AssignVariableOp_155"
  input: "^save/AssignVariableOp_156"
  input: "^save/AssignVariableOp_157"
  input: "^save/AssignVariableOp_158"
  input: "^save/AssignVariableOp_159"
  input: "^save/AssignVariableOp_16"
  input: "^save/AssignVariableOp_160"
  input: "^save/AssignVariableOp_161"
  input: "^save/AssignVariableOp_162"
  input: "^save/AssignVariableOp_163"
  input: "^save/AssignVariableOp_164"
  input: "^save/AssignVariableOp_165"
  input: "^save/AssignVariableOp_166"
  input: "^save/AssignVariableOp_167"
  input: "^save/AssignVariableOp_168"
  input: "^save/AssignVariableOp_169"
  input: "^save/AssignVariableOp_17"
  input: "^save/AssignVariableOp_170"
  input: "^save/AssignVariableOp_171"
  input: "^save/AssignVariableOp_172"
  input: "^save/AssignVariableOp_173"
  input: "^save/AssignVariableOp_174"
  input: "^save/AssignVariableOp_175"
  input: "^save/AssignVariableOp_176"
  input: "^save/AssignVariableOp_177"
  input: "^save/AssignVariableOp_178"
  input: "^save/AssignVariableOp_179"
  input: "^save/AssignVariableOp_18"
  input: "^save/AssignVariableOp_180"
  input: "^save/AssignVariableOp_181"
  input: "^save/AssignVariableOp_182"
  input: "^save/AssignVariableOp_183"
  input: "^save/AssignVariableOp_184"
  input: "^save/AssignVariableOp_185"
  input: "^save/AssignVariableOp_186"
  input: "^save/AssignVariableOp_187"
  input: "^save/AssignVariableOp_188"
  input: "^save/AssignVariableOp_189"
  input: "^save/AssignVariableOp_19"
  input: "^save/AssignVariableOp_190"
  input: "^save/AssignVariableOp_191"
  input: "^save/AssignVariableOp_192"
  input: "^save/AssignVariableOp_193"
  input: "^save/AssignVariableOp_194"
  input: "^save/AssignVariableOp_195"
  input: "^save/AssignVariableOp_196"
  input: "^save/AssignVariableOp_197"
  input: "^save/AssignVariableOp_198"
  input: "^save/AssignVariableOp_199"
  input: "^save/AssignVariableOp_2"
  input: "^save/AssignVariableOp_20"
  input: "^save/AssignVariableOp_200"
  input: "^save/AssignVariableOp_201"
  input: "^save/AssignVariableOp_202"
  input: "^save/AssignVariableOp_203"
  input: "^save/AssignVariableOp_204"
  input: "^save/AssignVariableOp_205"
  input: "^save/AssignVariableOp_206"
  input: "^save/AssignVariableOp_207"
  input: "^save/AssignVariableOp_208"
  input: "^save/AssignVariableOp_209"
  input: "^save/AssignVariableOp_21"
  input: "^save/AssignVariableOp_210"
  input: "^save/AssignVariableOp_211"
  input: "^save/AssignVariableOp_212"
  input: "^save/AssignVariableOp_213"
  input: "^save/AssignVariableOp_214"
  input: "^save/AssignVariableOp_215"
  input: "^save/AssignVariableOp_216"
  input: "^save/AssignVariableOp_217"
  input: "^save/AssignVariableOp_218"
  input: "^save/AssignVariableOp_219"
  input: "^save/AssignVariableOp_22"
  input: "^save/AssignVariableOp_220"
  input: "^save/AssignVariableOp_221"
  input: "^save/AssignVariableOp_222"
  input: "^save/AssignVariableOp_223"
  input: "^save/AssignVariableOp_224"
  input: "^save/AssignVariableOp_225"
  input: "^save/AssignVariableOp_226"
  input: "^save/AssignVariableOp_227"
  input: "^save/AssignVariableOp_228"
  input: "^save/AssignVariableOp_229"
  input: "^save/AssignVariableOp_23"
  input: "^save/AssignVariableOp_230"
  input: "^save/AssignVariableOp_231"
  input: "^save/AssignVariableOp_232"
  input: "^save/AssignVariableOp_233"
  input: "^save/AssignVariableOp_234"
  input: "^save/AssignVariableOp_235"
  input: "^save/AssignVariableOp_236"
  input: "^save/AssignVariableOp_237"
  input: "^save/AssignVariableOp_238"
  input: "^save/AssignVariableOp_239"
  input: "^save/AssignVariableOp_24"
  input: "^save/AssignVariableOp_240"
  input: "^save/AssignVariableOp_241"
  input: "^save/AssignVariableOp_242"
  input: "^save/AssignVariableOp_243"
  input: "^save/AssignVariableOp_244"
  input: "^save/AssignVariableOp_245"
  input: "^save/AssignVariableOp_246"
  input: "^save/AssignVariableOp_247"
  input: "^save/AssignVariableOp_248"
  input: "^save/AssignVariableOp_249"
  input: "^save/AssignVariableOp_25"
  input: "^save/AssignVariableOp_250"
  input: "^save/AssignVariableOp_251"
  input: "^save/AssignVariableOp_252"
  input: "^save/AssignVariableOp_253"
  input: "^save/AssignVariableOp_254"
  input: "^save/AssignVariableOp_255"
  input: "^save/AssignVariableOp_256"
  input: "^save/AssignVariableOp_257"
  input: "^save/AssignVariableOp_258"
  input: "^save/AssignVariableOp_259"
  input: "^save/AssignVariableOp_26"
  input: "^save/AssignVariableOp_260"
  input: "^save/AssignVariableOp_261"
  input: "^save/AssignVariableOp_262"
  input: "^save/AssignVariableOp_263"
  input: "^save/AssignVariableOp_264"
  input: "^save/AssignVariableOp_265"
  input: "^save/AssignVariableOp_266"
  input: "^save/AssignVariableOp_267"
  input: "^save/AssignVariableOp_268"
  input: "^save/AssignVariableOp_269"
  input: "^save/AssignVariableOp_27"
  input: "^save/AssignVariableOp_270"
  input: "^save/AssignVariableOp_271"
  input: "^save/AssignVariableOp_272"
  input: "^save/AssignVariableOp_273"
  input: "^save/AssignVariableOp_274"
  input: "^save/AssignVariableOp_275"
  input: "^save/AssignVariableOp_276"
  input: "^save/AssignVariableOp_277"
  input: "^save/AssignVariableOp_278"
  input: "^save/AssignVariableOp_279"
  input: "^save/AssignVariableOp_28"
  input: "^save/AssignVariableOp_280"
  input: "^save/AssignVariableOp_281"
  input: "^save/AssignVariableOp_282"
  input: "^save/AssignVariableOp_283"
  input: "^save/AssignVariableOp_284"
  input: "^save/AssignVariableOp_285"
  input: "^save/AssignVariableOp_286"
  input: "^save/AssignVariableOp_287"
  input: "^save/AssignVariableOp_288"
  input: "^save/AssignVariableOp_289"
  input: "^save/AssignVariableOp_29"
  input: "^save/AssignVariableOp_290"
  input: "^save/AssignVariableOp_291"
  input: "^save/AssignVariableOp_292"
  input: "^save/AssignVariableOp_293"
  input: "^save/AssignVariableOp_294"
  input: "^save/AssignVariableOp_295"
  input: "^save/AssignVariableOp_296"
  input: "^save/AssignVariableOp_297"
  input: "^save/AssignVariableOp_298"
  input: "^save/AssignVariableOp_299"
  input: "^save/AssignVariableOp_3"
  input: "^save/AssignVariableOp_30"
  input: "^save/AssignVariableOp_300"
  input: "^save/AssignVariableOp_301"
  input: "^save/AssignVariableOp_302"
  input: "^save/AssignVariableOp_303"
  input: "^save/AssignVariableOp_304"
  input: "^save/AssignVariableOp_305"
  input: "^save/AssignVariableOp_306"
  input: "^save/AssignVariableOp_307"
  input: "^save/AssignVariableOp_308"
  input: "^save/AssignVariableOp_309"
  input: "^save/AssignVariableOp_31"
  input: "^save/AssignVariableOp_310"
  input: "^save/AssignVariableOp_311"
  input: "^save/AssignVariableOp_312"
  input: "^save/AssignVariableOp_313"
  input: "^save/AssignVariableOp_314"
  input: "^save/AssignVariableOp_315"
  input: "^save/AssignVariableOp_316"
  input: "^save/AssignVariableOp_317"
  input: "^save/AssignVariableOp_318"
  input: "^save/AssignVariableOp_319"
  input: "^save/AssignVariableOp_32"
  input: "^save/AssignVariableOp_320"
  input: "^save/AssignVariableOp_321"
  input: "^save/AssignVariableOp_322"
  input: "^save/AssignVariableOp_323"
  input: "^save/AssignVariableOp_324"
  input: "^save/AssignVariableOp_325"
  input: "^save/AssignVariableOp_326"
  input: "^save/AssignVariableOp_327"
  input: "^save/AssignVariableOp_328"
  input: "^save/AssignVariableOp_329"
  input: "^save/AssignVariableOp_33"
  input: "^save/AssignVariableOp_330"
  input: "^save/AssignVariableOp_331"
  input: "^save/AssignVariableOp_332"
  input: "^save/AssignVariableOp_333"
  input: "^save/AssignVariableOp_334"
  input: "^save/AssignVariableOp_335"
  input: "^save/AssignVariableOp_336"
  input: "^save/AssignVariableOp_337"
  input: "^save/AssignVariableOp_338"
  input: "^save/AssignVariableOp_339"
  input: "^save/AssignVariableOp_34"
  input: "^save/AssignVariableOp_340"
  input: "^save/AssignVariableOp_341"
  input: "^save/AssignVariableOp_342"
  input: "^save/AssignVariableOp_343"
  input: "^save/AssignVariableOp_344"
  input: "^save/AssignVariableOp_345"
  input: "^save/AssignVariableOp_346"
  input: "^save/AssignVariableOp_347"
  input: "^save/AssignVariableOp_348"
  input: "^save/AssignVariableOp_349"
  input: "^save/AssignVariableOp_35"
  input: "^save/AssignVariableOp_350"
  input: "^save/AssignVariableOp_351"
  input: "^save/AssignVariableOp_352"
  input: "^save/AssignVariableOp_353"
  input: "^save/AssignVariableOp_354"
  input: "^save/AssignVariableOp_355"
  input: "^save/AssignVariableOp_356"
  input: "^save/AssignVariableOp_357"
  input: "^save/AssignVariableOp_358"
  input: "^save/AssignVariableOp_359"
  input: "^save/AssignVariableOp_36"
  input: "^save/AssignVariableOp_360"
  input: "^save/AssignVariableOp_361"
  input: "^save/AssignVariableOp_362"
  input: "^save/AssignVariableOp_363"
  input: "^save/AssignVariableOp_364"
  input: "^save/AssignVariableOp_365"
  input: "^save/AssignVariableOp_366"
  input: "^save/AssignVariableOp_367"
  input: "^save/AssignVariableOp_368"
  input: "^save/AssignVariableOp_369"
  input: "^save/AssignVariableOp_37"
  input: "^save/AssignVariableOp_370"
  input: "^save/AssignVariableOp_371"
  input: "^save/AssignVariableOp_372"
  input: "^save/AssignVariableOp_373"
  input: "^save/AssignVariableOp_374"
  input: "^save/AssignVariableOp_375"
  input: "^save/AssignVariableOp_376"
  input: "^save/AssignVariableOp_377"
  input: "^save/AssignVariableOp_378"
  input: "^save/AssignVariableOp_379"
  input: "^save/AssignVariableOp_38"
  input: "^save/AssignVariableOp_380"
  input: "^save/AssignVariableOp_381"
  input: "^save/AssignVariableOp_382"
  input: "^save/AssignVariableOp_383"
  input: "^save/AssignVariableOp_384"
  input: "^save/AssignVariableOp_385"
  input: "^save/AssignVariableOp_386"
  input: "^save/AssignVariableOp_387"
  input: "^save/AssignVariableOp_388"
  input: "^save/AssignVariableOp_389"
  input: "^save/AssignVariableOp_39"
  input: "^save/AssignVariableOp_390"
  input: "^save/AssignVariableOp_391"
  input: "^save/AssignVariableOp_392"
  input: "^save/AssignVariableOp_393"
  input: "^save/AssignVariableOp_394"
  input: "^save/AssignVariableOp_395"
  input: "^save/AssignVariableOp_396"
  input: "^save/AssignVariableOp_397"
  input: "^save/AssignVariableOp_398"
  input: "^save/AssignVariableOp_399"
  input: "^save/AssignVariableOp_4"
  input: "^save/AssignVariableOp_40"
  input: "^save/AssignVariableOp_400"
  input: "^save/AssignVariableOp_401"
  input: "^save/AssignVariableOp_402"
  input: "^save/AssignVariableOp_403"
  input: "^save/AssignVariableOp_404"
  input: "^save/AssignVariableOp_405"
  input: "^save/AssignVariableOp_406"
  input: "^save/AssignVariableOp_407"
  input: "^save/AssignVariableOp_408"
  input: "^save/AssignVariableOp_409"
  input: "^save/AssignVariableOp_41"
  input: "^save/AssignVariableOp_410"
  input: "^save/AssignVariableOp_411"
  input: "^save/AssignVariableOp_412"
  input: "^save/AssignVariableOp_413"
  input: "^save/AssignVariableOp_414"
  input: "^save/AssignVariableOp_415"
  input: "^save/AssignVariableOp_416"
  input: "^save/AssignVariableOp_417"
  input: "^save/AssignVariableOp_418"
  input: "^save/AssignVariableOp_419"
  input: "^save/AssignVariableOp_42"
  input: "^save/AssignVariableOp_420"
  input: "^save/AssignVariableOp_421"
  input: "^save/AssignVariableOp_422"
  input: "^save/AssignVariableOp_423"
  input: "^save/AssignVariableOp_424"
  input: "^save/AssignVariableOp_425"
  input: "^save/AssignVariableOp_426"
  input: "^save/AssignVariableOp_427"
  input: "^save/AssignVariableOp_428"
  input: "^save/AssignVariableOp_429"
  input: "^save/AssignVariableOp_43"
  input: "^save/AssignVariableOp_430"
  input: "^save/AssignVariableOp_431"
  input: "^save/AssignVariableOp_432"
  input: "^save/AssignVariableOp_433"
  input: "^save/AssignVariableOp_434"
  input: "^save/AssignVariableOp_435"
  input: "^save/AssignVariableOp_436"
  input: "^save/AssignVariableOp_437"
  input: "^save/AssignVariableOp_438"
  input: "^save/AssignVariableOp_439"
  input: "^save/AssignVariableOp_44"
  input: "^save/AssignVariableOp_440"
  input: "^save/AssignVariableOp_441"
  input: "^save/AssignVariableOp_442"
  input: "^save/AssignVariableOp_443"
  input: "^save/AssignVariableOp_444"
  input: "^save/AssignVariableOp_445"
  input: "^save/AssignVariableOp_446"
  input: "^save/AssignVariableOp_447"
  input: "^save/AssignVariableOp_448"
  input: "^save/AssignVariableOp_449"
  input: "^save/AssignVariableOp_45"
  input: "^save/AssignVariableOp_450"
  input: "^save/AssignVariableOp_451"
  input: "^save/AssignVariableOp_452"
  input: "^save/AssignVariableOp_453"
  input: "^save/AssignVariableOp_454"
  input: "^save/AssignVariableOp_455"
  input: "^save/AssignVariableOp_456"
  input: "^save/AssignVariableOp_457"
  input: "^save/AssignVariableOp_458"
  input: "^save/AssignVariableOp_459"
  input: "^save/AssignVariableOp_46"
  input: "^save/AssignVariableOp_460"
  input: "^save/AssignVariableOp_461"
  input: "^save/AssignVariableOp_462"
  input: "^save/AssignVariableOp_463"
  input: "^save/AssignVariableOp_464"
  input: "^save/AssignVariableOp_465"
  input: "^save/AssignVariableOp_466"
  input: "^save/AssignVariableOp_467"
  input: "^save/AssignVariableOp_468"
  input: "^save/AssignVariableOp_469"
  input: "^save/AssignVariableOp_47"
  input: "^save/AssignVariableOp_470"
  input: "^save/AssignVariableOp_471"
  input: "^save/AssignVariableOp_472"
  input: "^save/AssignVariableOp_473"
  input: "^save/AssignVariableOp_474"
  input: "^save/AssignVariableOp_475"
  input: "^save/AssignVariableOp_476"
  input: "^save/AssignVariableOp_477"
  input: "^save/AssignVariableOp_478"
  input: "^save/AssignVariableOp_479"
  input: "^save/AssignVariableOp_48"
  input: "^save/AssignVariableOp_480"
  input: "^save/AssignVariableOp_481"
  input: "^save/AssignVariableOp_482"
  input: "^save/AssignVariableOp_483"
  input: "^save/AssignVariableOp_484"
  input: "^save/AssignVariableOp_485"
  input: "^save/AssignVariableOp_486"
  input: "^save/AssignVariableOp_487"
  input: "^save/AssignVariableOp_488"
  input: "^save/AssignVariableOp_489"
  input: "^save/AssignVariableOp_49"
  input: "^save/AssignVariableOp_490"
  input: "^save/AssignVariableOp_491"
  input: "^save/AssignVariableOp_492"
  input: "^save/AssignVariableOp_493"
  input: "^save/AssignVariableOp_494"
  input: "^save/AssignVariableOp_495"
  input: "^save/AssignVariableOp_496"
  input: "^save/AssignVariableOp_497"
  input: "^save/AssignVariableOp_498"
  input: "^save/AssignVariableOp_499"
  input: "^save/AssignVariableOp_5"
  input: "^save/AssignVariableOp_50"
  input: "^save/AssignVariableOp_500"
  input: "^save/AssignVariableOp_501"
  input: "^save/AssignVariableOp_502"
  input: "^save/AssignVariableOp_503"
  input: "^save/AssignVariableOp_504"
  input: "^save/AssignVariableOp_505"
  input: "^save/AssignVariableOp_506"
  input: "^save/AssignVariableOp_507"
  input: "^save/AssignVariableOp_508"
  input: "^save/AssignVariableOp_509"
  input: "^save/AssignVariableOp_51"
  input: "^save/AssignVariableOp_510"
  input: "^save/AssignVariableOp_511"
  input: "^save/AssignVariableOp_512"
  input: "^save/AssignVariableOp_513"
  input: "^save/AssignVariableOp_514"
  input: "^save/AssignVariableOp_515"
  input: "^save/AssignVariableOp_516"
  input: "^save/AssignVariableOp_517"
  input: "^save/AssignVariableOp_518"
  input: "^save/AssignVariableOp_519"
  input: "^save/AssignVariableOp_52"
  input: "^save/AssignVariableOp_520"
  input: "^save/AssignVariableOp_521"
  input: "^save/AssignVariableOp_522"
  input: "^save/AssignVariableOp_523"
  input: "^save/AssignVariableOp_524"
  input: "^save/AssignVariableOp_525"
  input: "^save/AssignVariableOp_526"
  input: "^save/AssignVariableOp_527"
  input: "^save/AssignVariableOp_528"
  input: "^save/AssignVariableOp_529"
  input: "^save/AssignVariableOp_53"
  input: "^save/AssignVariableOp_530"
  input: "^save/AssignVariableOp_531"
  input: "^save/AssignVariableOp_532"
  input: "^save/AssignVariableOp_533"
  input: "^save/AssignVariableOp_534"
  input: "^save/AssignVariableOp_535"
  input: "^save/AssignVariableOp_536"
  input: "^save/AssignVariableOp_537"
  input: "^save/AssignVariableOp_538"
  input: "^save/AssignVariableOp_539"
  input: "^save/AssignVariableOp_54"
  input: "^save/AssignVariableOp_540"
  input: "^save/AssignVariableOp_541"
  input: "^save/AssignVariableOp_542"
  input: "^save/AssignVariableOp_543"
  input: "^save/AssignVariableOp_544"
  input: "^save/AssignVariableOp_545"
  input: "^save/AssignVariableOp_546"
  input: "^save/AssignVariableOp_547"
  input: "^save/AssignVariableOp_548"
  input: "^save/AssignVariableOp_549"
  input: "^save/AssignVariableOp_55"
  input: "^save/AssignVariableOp_550"
  input: "^save/AssignVariableOp_551"
  input: "^save/AssignVariableOp_552"
  input: "^save/AssignVariableOp_553"
  input: "^save/AssignVariableOp_554"
  input: "^save/AssignVariableOp_555"
  input: "^save/AssignVariableOp_556"
  input: "^save/AssignVariableOp_557"
  input: "^save/AssignVariableOp_558"
  input: "^save/AssignVariableOp_559"
  input: "^save/AssignVariableOp_56"
  input: "^save/AssignVariableOp_560"
  input: "^save/AssignVariableOp_561"
  input: "^save/AssignVariableOp_562"
  input: "^save/AssignVariableOp_563"
  input: "^save/AssignVariableOp_564"
  input: "^save/AssignVariableOp_565"
  input: "^save/AssignVariableOp_566"
  input: "^save/AssignVariableOp_567"
  input: "^save/AssignVariableOp_568"
  input: "^save/AssignVariableOp_569"
  input: "^save/AssignVariableOp_57"
  input: "^save/AssignVariableOp_570"
  input: "^save/AssignVariableOp_571"
  input: "^save/AssignVariableOp_572"
  input: "^save/AssignVariableOp_573"
  input: "^save/AssignVariableOp_574"
  input: "^save/AssignVariableOp_575"
  input: "^save/AssignVariableOp_576"
  input: "^save/AssignVariableOp_577"
  input: "^save/AssignVariableOp_578"
  input: "^save/AssignVariableOp_579"
  input: "^save/AssignVariableOp_58"
  input: "^save/AssignVariableOp_580"
  input: "^save/AssignVariableOp_581"
  input: "^save/AssignVariableOp_582"
  input: "^save/AssignVariableOp_583"
  input: "^save/AssignVariableOp_584"
  input: "^save/AssignVariableOp_585"
  input: "^save/AssignVariableOp_586"
  input: "^save/AssignVariableOp_587"
  input: "^save/AssignVariableOp_588"
  input: "^save/AssignVariableOp_589"
  input: "^save/AssignVariableOp_59"
  input: "^save/AssignVariableOp_590"
  input: "^save/AssignVariableOp_591"
  input: "^save/AssignVariableOp_592"
  input: "^save/AssignVariableOp_593"
  input: "^save/AssignVariableOp_594"
  input: "^save/AssignVariableOp_595"
  input: "^save/AssignVariableOp_596"
  input: "^save/AssignVariableOp_597"
  input: "^save/AssignVariableOp_598"
  input: "^save/AssignVariableOp_599"
  input: "^save/AssignVariableOp_6"
  input: "^save/AssignVariableOp_60"
  input: "^save/AssignVariableOp_600"
  input: "^save/AssignVariableOp_601"
  input: "^save/AssignVariableOp_602"
  input: "^save/AssignVariableOp_603"
  input: "^save/AssignVariableOp_604"
  input: "^save/AssignVariableOp_605"
  input: "^save/AssignVariableOp_606"
  input: "^save/AssignVariableOp_607"
  input: "^save/AssignVariableOp_608"
  input: "^save/AssignVariableOp_609"
  input: "^save/AssignVariableOp_61"
  input: "^save/AssignVariableOp_610"
  input: "^save/AssignVariableOp_611"
  input: "^save/AssignVariableOp_612"
  input: "^save/AssignVariableOp_613"
  input: "^save/AssignVariableOp_614"
  input: "^save/AssignVariableOp_615"
  input: "^save/AssignVariableOp_616"
  input: "^save/AssignVariableOp_617"
  input: "^save/AssignVariableOp_618"
  input: "^save/AssignVariableOp_619"
  input: "^save/AssignVariableOp_62"
  input: "^save/AssignVariableOp_620"
  input: "^save/AssignVariableOp_621"
  input: "^save/AssignVariableOp_622"
  input: "^save/AssignVariableOp_623"
  input: "^save/AssignVariableOp_624"
  input: "^save/AssignVariableOp_625"
  input: "^save/AssignVariableOp_626"
  input: "^save/AssignVariableOp_627"
  input: "^save/AssignVariableOp_628"
  input: "^save/AssignVariableOp_629"
  input: "^save/AssignVariableOp_63"
  input: "^save/AssignVariableOp_630"
  input: "^save/AssignVariableOp_631"
  input: "^save/AssignVariableOp_632"
  input: "^save/AssignVariableOp_633"
  input: "^save/AssignVariableOp_634"
  input: "^save/AssignVariableOp_635"
  input: "^save/AssignVariableOp_636"
  input: "^save/AssignVariableOp_637"
  input: "^save/AssignVariableOp_638"
  input: "^save/AssignVariableOp_639"
  input: "^save/AssignVariableOp_64"
  input: "^save/AssignVariableOp_640"
  input: "^save/AssignVariableOp_641"
  input: "^save/AssignVariableOp_642"
  input: "^save/AssignVariableOp_643"
  input: "^save/AssignVariableOp_644"
  input: "^save/AssignVariableOp_645"
  input: "^save/AssignVariableOp_646"
  input: "^save/AssignVariableOp_647"
  input: "^save/AssignVariableOp_648"
  input: "^save/AssignVariableOp_649"
  input: "^save/AssignVariableOp_65"
  input: "^save/AssignVariableOp_650"
  input: "^save/AssignVariableOp_651"
  input: "^save/AssignVariableOp_652"
  input: "^save/AssignVariableOp_653"
  input: "^save/AssignVariableOp_654"
  input: "^save/AssignVariableOp_655"
  input: "^save/AssignVariableOp_656"
  input: "^save/AssignVariableOp_657"
  input: "^save/AssignVariableOp_658"
  input: "^save/AssignVariableOp_659"
  input: "^save/AssignVariableOp_66"
  input: "^save/AssignVariableOp_660"
  input: "^save/AssignVariableOp_661"
  input: "^save/AssignVariableOp_662"
  input: "^save/AssignVariableOp_663"
  input: "^save/AssignVariableOp_664"
  input: "^save/AssignVariableOp_665"
  input: "^save/AssignVariableOp_666"
  input: "^save/AssignVariableOp_667"
  input: "^save/AssignVariableOp_668"
  input: "^save/AssignVariableOp_669"
  input: "^save/AssignVariableOp_67"
  input: "^save/AssignVariableOp_670"
  input: "^save/AssignVariableOp_671"
  input: "^save/AssignVariableOp_672"
  input: "^save/AssignVariableOp_673"
  input: "^save/AssignVariableOp_674"
  input: "^save/AssignVariableOp_675"
  input: "^save/AssignVariableOp_676"
  input: "^save/AssignVariableOp_677"
  input: "^save/AssignVariableOp_678"
  input: "^save/AssignVariableOp_679"
  input: "^save/AssignVariableOp_68"
  input: "^save/AssignVariableOp_680"
  input: "^save/AssignVariableOp_681"
  input: "^save/AssignVariableOp_682"
  input: "^save/AssignVariableOp_683"
  input: "^save/AssignVariableOp_684"
  input: "^save/AssignVariableOp_685"
  input: "^save/AssignVariableOp_686"
  input: "^save/AssignVariableOp_687"
  input: "^save/AssignVariableOp_688"
  input: "^save/AssignVariableOp_689"
  input: "^save/AssignVariableOp_69"
  input: "^save/AssignVariableOp_690"
  input: "^save/AssignVariableOp_691"
  input: "^save/AssignVariableOp_692"
  input: "^save/AssignVariableOp_693"
  input: "^save/AssignVariableOp_694"
  input: "^save/AssignVariableOp_695"
  input: "^save/AssignVariableOp_696"
  input: "^save/AssignVariableOp_697"
  input: "^save/AssignVariableOp_698"
  input: "^save/AssignVariableOp_699"
  input: "^save/AssignVariableOp_7"
  input: "^save/AssignVariableOp_70"
  input: "^save/AssignVariableOp_700"
  input: "^save/AssignVariableOp_701"
  input: "^save/AssignVariableOp_702"
  input: "^save/AssignVariableOp_703"
  input: "^save/AssignVariableOp_704"
  input: "^save/AssignVariableOp_705"
  input: "^save/AssignVariableOp_706"
  input: "^save/AssignVariableOp_707"
  input: "^save/AssignVariableOp_708"
  input: "^save/AssignVariableOp_709"
  input: "^save/AssignVariableOp_71"
  input: "^save/AssignVariableOp_710"
  input: "^save/AssignVariableOp_711"
  input: "^save/AssignVariableOp_712"
  input: "^save/AssignVariableOp_713"
  input: "^save/AssignVariableOp_714"
  input: "^save/AssignVariableOp_715"
  input: "^save/AssignVariableOp_716"
  input: "^save/AssignVariableOp_717"
  input: "^save/AssignVariableOp_718"
  input: "^save/AssignVariableOp_719"
  input: "^save/AssignVariableOp_72"
  input: "^save/AssignVariableOp_720"
  input: "^save/AssignVariableOp_721"
  input: "^save/AssignVariableOp_722"
  input: "^save/AssignVariableOp_723"
  input: "^save/AssignVariableOp_724"
  input: "^save/AssignVariableOp_725"
  input: "^save/AssignVariableOp_726"
  input: "^save/AssignVariableOp_727"
  input: "^save/AssignVariableOp_728"
  input: "^save/AssignVariableOp_729"
  input: "^save/AssignVariableOp_73"
  input: "^save/AssignVariableOp_730"
  input: "^save/AssignVariableOp_731"
  input: "^save/AssignVariableOp_732"
  input: "^save/AssignVariableOp_733"
  input: "^save/AssignVariableOp_734"
  input: "^save/AssignVariableOp_735"
  input: "^save/AssignVariableOp_736"
  input: "^save/AssignVariableOp_737"
  input: "^save/AssignVariableOp_738"
  input: "^save/AssignVariableOp_739"
  input: "^save/AssignVariableOp_74"
  input: "^save/AssignVariableOp_740"
  input: "^save/AssignVariableOp_741"
  input: "^save/AssignVariableOp_742"
  input: "^save/AssignVariableOp_743"
  input: "^save/AssignVariableOp_744"
  input: "^save/AssignVariableOp_745"
  input: "^save/AssignVariableOp_746"
  input: "^save/AssignVariableOp_747"
  input: "^save/AssignVariableOp_748"
  input: "^save/AssignVariableOp_749"
  input: "^save/AssignVariableOp_75"
  input: "^save/AssignVariableOp_750"
  input: "^save/AssignVariableOp_751"
  input: "^save/AssignVariableOp_752"
  input: "^save/AssignVariableOp_753"
  input: "^save/AssignVariableOp_754"
  input: "^save/AssignVariableOp_755"
  input: "^save/AssignVariableOp_756"
  input: "^save/AssignVariableOp_757"
  input: "^save/AssignVariableOp_758"
  input: "^save/AssignVariableOp_759"
  input: "^save/AssignVariableOp_76"
  input: "^save/AssignVariableOp_760"
  input: "^save/AssignVariableOp_761"
  input: "^save/AssignVariableOp_762"
  input: "^save/AssignVariableOp_763"
  input: "^save/AssignVariableOp_764"
  input: "^save/AssignVariableOp_765"
  input: "^save/AssignVariableOp_766"
  input: "^save/AssignVariableOp_767"
  input: "^save/AssignVariableOp_768"
  input: "^save/AssignVariableOp_769"
  input: "^save/AssignVariableOp_77"
  input: "^save/AssignVariableOp_770"
  input: "^save/AssignVariableOp_771"
  input: "^save/AssignVariableOp_772"
  input: "^save/AssignVariableOp_773"
  input: "^save/AssignVariableOp_774"
  input: "^save/AssignVariableOp_775"
  input: "^save/AssignVariableOp_776"
  input: "^save/AssignVariableOp_777"
  input: "^save/AssignVariableOp_778"
  input: "^save/AssignVariableOp_779"
  input: "^save/AssignVariableOp_78"
  input: "^save/AssignVariableOp_780"
  input: "^save/AssignVariableOp_781"
  input: "^save/AssignVariableOp_782"
  input: "^save/AssignVariableOp_783"
  input: "^save/AssignVariableOp_784"
  input: "^save/AssignVariableOp_785"
  input: "^save/AssignVariableOp_786"
  input: "^save/AssignVariableOp_787"
  input: "^save/AssignVariableOp_788"
  input: "^save/AssignVariableOp_789"
  input: "^save/AssignVariableOp_79"
  input: "^save/AssignVariableOp_790"
  input: "^save/AssignVariableOp_791"
  input: "^save/AssignVariableOp_792"
  input: "^save/AssignVariableOp_793"
  input: "^save/AssignVariableOp_794"
  input: "^save/AssignVariableOp_795"
  input: "^save/AssignVariableOp_796"
  input: "^save/AssignVariableOp_797"
  input: "^save/AssignVariableOp_798"
  input: "^save/AssignVariableOp_799"
  input: "^save/AssignVariableOp_8"
  input: "^save/AssignVariableOp_80"
  input: "^save/AssignVariableOp_800"
  input: "^save/AssignVariableOp_801"
  input: "^save/AssignVariableOp_802"
  input: "^save/AssignVariableOp_803"
  input: "^save/AssignVariableOp_804"
  input: "^save/AssignVariableOp_805"
  input: "^save/AssignVariableOp_806"
  input: "^save/AssignVariableOp_807"
  input: "^save/AssignVariableOp_808"
  input: "^save/AssignVariableOp_809"
  input: "^save/AssignVariableOp_81"
  input: "^save/AssignVariableOp_810"
  input: "^save/AssignVariableOp_811"
  input: "^save/AssignVariableOp_812"
  input: "^save/AssignVariableOp_813"
  input: "^save/AssignVariableOp_814"
  input: "^save/AssignVariableOp_815"
  input: "^save/AssignVariableOp_816"
  input: "^save/AssignVariableOp_817"
  input: "^save/AssignVariableOp_818"
  input: "^save/AssignVariableOp_819"
  input: "^save/AssignVariableOp_82"
  input: "^save/AssignVariableOp_820"
  input: "^save/AssignVariableOp_821"
  input: "^save/AssignVariableOp_822"
  input: "^save/AssignVariableOp_823"
  input: "^save/AssignVariableOp_824"
  input: "^save/AssignVariableOp_825"
  input: "^save/AssignVariableOp_826"
  input: "^save/AssignVariableOp_827"
  input: "^save/AssignVariableOp_828"
  input: "^save/AssignVariableOp_829"
  input: "^save/AssignVariableOp_83"
  input: "^save/AssignVariableOp_830"
  input: "^save/AssignVariableOp_831"
  input: "^save/AssignVariableOp_832"
  input: "^save/AssignVariableOp_833"
  input: "^save/AssignVariableOp_834"
  input: "^save/AssignVariableOp_835"
  input: "^save/AssignVariableOp_836"
  input: "^save/AssignVariableOp_837"
  input: "^save/AssignVariableOp_838"
  input: "^save/AssignVariableOp_839"
  input: "^save/AssignVariableOp_84"
  input: "^save/AssignVariableOp_840"
  input: "^save/AssignVariableOp_841"
  input: "^save/AssignVariableOp_842"
  input: "^save/AssignVariableOp_843"
  input: "^save/AssignVariableOp_844"
  input: "^save/AssignVariableOp_845"
  input: "^save/AssignVariableOp_846"
  input: "^save/AssignVariableOp_847"
  input: "^save/AssignVariableOp_848"
  input: "^save/AssignVariableOp_849"
  input: "^save/AssignVariableOp_85"
  input: "^save/AssignVariableOp_850"
  input: "^save/AssignVariableOp_851"
  input: "^save/AssignVariableOp_852"
  input: "^save/AssignVariableOp_853"
  input: "^save/AssignVariableOp_854"
  input: "^save/AssignVariableOp_855"
  input: "^save/AssignVariableOp_856"
  input: "^save/AssignVariableOp_857"
  input: "^save/AssignVariableOp_858"
  input: "^save/AssignVariableOp_859"
  input: "^save/AssignVariableOp_86"
  input: "^save/AssignVariableOp_860"
  input: "^save/AssignVariableOp_861"
  input: "^save/AssignVariableOp_862"
  input: "^save/AssignVariableOp_863"
  input: "^save/AssignVariableOp_864"
  input: "^save/AssignVariableOp_865"
  input: "^save/AssignVariableOp_866"
  input: "^save/AssignVariableOp_867"
  input: "^save/AssignVariableOp_868"
  input: "^save/AssignVariableOp_869"
  input: "^save/AssignVariableOp_87"
  input: "^save/AssignVariableOp_870"
  input: "^save/AssignVariableOp_871"
  input: "^save/AssignVariableOp_872"
  input: "^save/AssignVariableOp_873"
  input: "^save/AssignVariableOp_874"
  input: "^save/AssignVariableOp_875"
  input: "^save/AssignVariableOp_876"
  input: "^save/AssignVariableOp_877"
  input: "^save/AssignVariableOp_878"
  input: "^save/AssignVariableOp_879"
  input: "^save/AssignVariableOp_88"
  input: "^save/AssignVariableOp_880"
  input: "^save/AssignVariableOp_881"
  input: "^save/AssignVariableOp_882"
  input: "^save/AssignVariableOp_883"
  input: "^save/AssignVariableOp_884"
  input: "^save/AssignVariableOp_885"
  input: "^save/AssignVariableOp_886"
  input: "^save/AssignVariableOp_887"
  input: "^save/AssignVariableOp_888"
  input: "^save/AssignVariableOp_889"
  input: "^save/AssignVariableOp_89"
  input: "^save/AssignVariableOp_890"
  input: "^save/AssignVariableOp_891"
  input: "^save/AssignVariableOp_892"
  input: "^save/AssignVariableOp_893"
  input: "^save/AssignVariableOp_894"
  input: "^save/AssignVariableOp_895"
  input: "^save/AssignVariableOp_896"
  input: "^save/AssignVariableOp_897"
  input: "^save/AssignVariableOp_898"
  input: "^save/AssignVariableOp_899"
  input: "^save/AssignVariableOp_9"
  input: "^save/AssignVariableOp_90"
  input: "^save/AssignVariableOp_900"
  input: "^save/AssignVariableOp_901"
  input: "^save/AssignVariableOp_902"
  input: "^save/AssignVariableOp_903"
  input: "^save/AssignVariableOp_904"
  input: "^save/AssignVariableOp_905"
  input: "^save/AssignVariableOp_906"
  input: "^save/AssignVariableOp_907"
  input: "^save/AssignVariableOp_908"
  input: "^save/AssignVariableOp_909"
  input: "^save/AssignVariableOp_91"
  input: "^save/AssignVariableOp_910"
  input: "^save/AssignVariableOp_911"
  input: "^save/AssignVariableOp_912"
  input: "^save/AssignVariableOp_913"
  input: "^save/AssignVariableOp_914"
  input: "^save/AssignVariableOp_915"
  input: "^save/AssignVariableOp_916"
  input: "^save/AssignVariableOp_917"
  input: "^save/AssignVariableOp_918"
  input: "^save/AssignVariableOp_919"
  input: "^save/AssignVariableOp_92"
  input: "^save/AssignVariableOp_920"
  input: "^save/AssignVariableOp_921"
  input: "^save/AssignVariableOp_922"
  input: "^save/AssignVariableOp_923"
  input: "^save/AssignVariableOp_924"
  input: "^save/AssignVariableOp_925"
  input: "^save/AssignVariableOp_926"
  input: "^save/AssignVariableOp_927"
  input: "^save/AssignVariableOp_928"
  input: "^save/AssignVariableOp_929"
  input: "^save/AssignVariableOp_93"
  input: "^save/AssignVariableOp_930"
  input: "^save/AssignVariableOp_931"
  input: "^save/AssignVariableOp_932"
  input: "^save/AssignVariableOp_933"
  input: "^save/AssignVariableOp_934"
  input: "^save/AssignVariableOp_935"
  input: "^save/AssignVariableOp_936"
  input: "^save/AssignVariableOp_937"
  input: "^save/AssignVariableOp_938"
  input: "^save/AssignVariableOp_939"
  input: "^save/AssignVariableOp_94"
  input: "^save/AssignVariableOp_940"
  input: "^save/AssignVariableOp_941"
  input: "^save/AssignVariableOp_942"
  input: "^save/AssignVariableOp_943"
  input: "^save/AssignVariableOp_944"
  input: "^save/AssignVariableOp_945"
  input: "^save/AssignVariableOp_946"
  input: "^save/AssignVariableOp_947"
  input: "^save/AssignVariableOp_948"
  input: "^save/AssignVariableOp_949"
  input: "^save/AssignVariableOp_95"
  input: "^save/AssignVariableOp_950"
  input: "^save/AssignVariableOp_951"
  input: "^save/AssignVariableOp_952"
  input: "^save/AssignVariableOp_953"
  input: "^save/AssignVariableOp_954"
  input: "^save/AssignVariableOp_955"
  input: "^save/AssignVariableOp_956"
  input: "^save/AssignVariableOp_957"
  input: "^save/AssignVariableOp_958"
  input: "^save/AssignVariableOp_959"
  input: "^save/AssignVariableOp_96"
  input: "^save/AssignVariableOp_960"
  input: "^save/AssignVariableOp_961"
  input: "^save/AssignVariableOp_962"
  input: "^save/AssignVariableOp_963"
  input: "^save/AssignVariableOp_964"
  input: "^save/AssignVariableOp_965"
  input: "^save/AssignVariableOp_966"
  input: "^save/AssignVariableOp_967"
  input: "^save/AssignVariableOp_968"
  input: "^save/AssignVariableOp_969"
  input: "^save/AssignVariableOp_97"
  input: "^save/AssignVariableOp_970"
  input: "^save/AssignVariableOp_971"
  input: "^save/AssignVariableOp_972"
  input: "^save/AssignVariableOp_973"
  input: "^save/AssignVariableOp_974"
  input: "^save/AssignVariableOp_975"
  input: "^save/AssignVariableOp_976"
  input: "^save/AssignVariableOp_977"
  input: "^save/AssignVariableOp_978"
  input: "^save/AssignVariableOp_979"
  input: "^save/AssignVariableOp_98"
  input: "^save/AssignVariableOp_980"
  input: "^save/AssignVariableOp_981"
  input: "^save/AssignVariableOp_982"
  input: "^save/AssignVariableOp_983"
  input: "^save/AssignVariableOp_984"
  input: "^save/AssignVariableOp_985"
  input: "^save/AssignVariableOp_986"
  input: "^save/AssignVariableOp_987"
  input: "^save/AssignVariableOp_988"
  input: "^save/AssignVariableOp_989"
  input: "^save/AssignVariableOp_99"
  input: "^save/AssignVariableOp_990"
  input: "^save/AssignVariableOp_991"
  input: "^save/AssignVariableOp_992"
  input: "^save/AssignVariableOp_993"
  input: "^save/AssignVariableOp_994"
  input: "^save/AssignVariableOp_995"
  input: "^save/AssignVariableOp_996"
  input: "^save/AssignVariableOp_997"
  input: "^save/AssignVariableOp_998"
  input: "^save/AssignVariableOp_999"
}
node {
  name: "save/restore_all"
  op: "NoOp"
  input: "^save/restore_shard"
}
library {
  function {
    signature {
      name: "__inference_Dataset_flat_map_read_one_file_41"
      input_arg {
        name: "args_0"
        type: DT_STRING
      }
      output_arg {
        name: "identity"
        type: DT_VARIANT
      }
      is_stateful: true
      control_output: "TFRecordDataset"
    }
    node_def {
      name: "compression_type"
      op: "Const"
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
            }
          }
        }
      }
      attr {
        key: "dtype"
        value {
          type: DT_STRING
        }
      }
      attr {
        key: "value"
        value {
          tensor {
            dtype: DT_STRING
            tensor_shape {
            }
            string_val: ""
          }
        }
      }
      experimental_debug_info {
        original_node_names: "compression_type"
      }
    }
    node_def {
      name: "buffer_size"
      op: "Const"
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
            }
          }
        }
      }
      attr {
        key: "dtype"
        value {
          type: DT_INT64
        }
      }
      attr {
        key: "value"
        value {
          tensor {
            dtype: DT_INT64
            tensor_shape {
            }
            int64_val: 262144
          }
        }
      }
      experimental_debug_info {
        original_node_names: "buffer_size"
      }
    }
    node_def {
      name: "TFRecordDataset"
      op: "TFRecordDataset"
      input: "args_0"
      input: "compression_type:output:0"
      input: "buffer_size:output:0"
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "TFRecordDataset"
      }
    }
    node_def {
      name: "Identity"
      op: "Identity"
      input: "TFRecordDataset:handle:0"
      input: "^NoOp"
      attr {
        key: "T"
        value {
          type: DT_VARIANT
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "Identity"
      }
    }
    node_def {
      name: "NoOp"
      op: "NoOp"
      input: "^TFRecordDataset"
      attr {
        key: "_acd_function_control_output"
        value {
          b: true
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
          }
        }
      }
      experimental_debug_info {
        original_node_names: "NoOp"
      }
    }
    ret {
      key: "identity"
      value: "Identity:output:0"
    }
    attr {
      key: "_input_shapes"
      value {
        list {
          shape {
          }
        }
      }
    }
    attr {
      key: "_tf_data_function"
      value {
        b: true
      }
    }
    control_ret {
      key: "TFRecordDataset"
      value: "TFRecordDataset"
    }
    arg_attr {
      value {
        attr {
          key: "_output_shapes"
          value {
            list {
              shape {
              }
            }
          }
        }
        attr {
          key: "_user_specified_name"
          value {
            s: "args_0"
          }
        }
      }
    }
  }
  function {
    signature {
      name: "__inference_tf_data_experimental_map_and_batch_lambda_72"
      input_arg {
        name: "args_0"
        type: DT_STRING
      }
      output_arg {
        name: "identity"
        type: DT_INT32
      }
      output_arg {
        name: "identity_1"
        type: DT_INT32
      }
      output_arg {
        name: "identity_2"
        type: DT_INT32
      }
    }
    node_def {
      name: "ParseSingleExample/ParseExample/Const"
      op: "Const"
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
              }
            }
          }
        }
      }
      attr {
        key: "dtype"
        value {
          type: DT_INT64
        }
      }
      attr {
        key: "value"
        value {
          tensor {
            dtype: DT_INT64
            tensor_shape {
              dim {
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "ParseSingleExample/ParseExample/Const"
      }
    }
    node_def {
      name: "ParseSingleExample/ParseExample/Const_1"
      op: "Const"
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
              }
            }
          }
        }
      }
      attr {
        key: "dtype"
        value {
          type: DT_INT64
        }
      }
      attr {
        key: "value"
        value {
          tensor {
            dtype: DT_INT64
            tensor_shape {
              dim {
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "ParseSingleExample/ParseExample/Const_1"
      }
    }
    node_def {
      name: "ParseSingleExample/ParseExample/Const_2"
      op: "Const"
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
              }
            }
          }
        }
      }
      attr {
        key: "dtype"
        value {
          type: DT_INT64
        }
      }
      attr {
        key: "value"
        value {
          tensor {
            dtype: DT_INT64
            tensor_shape {
              dim {
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "ParseSingleExample/ParseExample/Const_2"
      }
    }
    node_def {
      name: "ParseSingleExample/ParseExample/ParseExampleV2/names"
      op: "Const"
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
              }
            }
          }
        }
      }
      attr {
        key: "dtype"
        value {
          type: DT_STRING
        }
      }
      attr {
        key: "value"
        value {
          tensor {
            dtype: DT_STRING
            tensor_shape {
              dim {
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "ParseSingleExample/ParseExample/ParseExampleV2/names"
      }
    }
    node_def {
      name: "ParseSingleExample/ParseExample/ParseExampleV2/sparse_keys"
      op: "Const"
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
              }
            }
          }
        }
      }
      attr {
        key: "dtype"
        value {
          type: DT_STRING
        }
      }
      attr {
        key: "value"
        value {
          tensor {
            dtype: DT_STRING
            tensor_shape {
              dim {
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "ParseSingleExample/ParseExample/ParseExampleV2/sparse_keys"
      }
    }
    node_def {
      name: "ParseSingleExample/ParseExample/ParseExampleV2/dense_keys"
      op: "Const"
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
                size: 3
              }
            }
          }
        }
      }
      attr {
        key: "dtype"
        value {
          type: DT_STRING
        }
      }
      attr {
        key: "value"
        value {
          tensor {
            dtype: DT_STRING
            tensor_shape {
              dim {
                size: 3
              }
            }
            string_val: "input_ids"
            string_val: "input_mask"
            string_val: "segment_ids"
          }
        }
      }
      experimental_debug_info {
        original_node_names: "ParseSingleExample/ParseExample/ParseExampleV2/dense_keys"
      }
    }
    node_def {
      name: "ParseSingleExample/ParseExample/ParseExampleV2/ragged_keys"
      op: "Const"
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
              }
            }
          }
        }
      }
      attr {
        key: "dtype"
        value {
          type: DT_STRING
        }
      }
      attr {
        key: "value"
        value {
          tensor {
            dtype: DT_STRING
            tensor_shape {
              dim {
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "ParseSingleExample/ParseExample/ParseExampleV2/ragged_keys"
      }
    }
    node_def {
      name: "ParseSingleExample/ParseExample/ParseExampleV2"
      op: "ParseExampleV2"
      input: "args_0"
      input: "ParseSingleExample/ParseExample/ParseExampleV2/names:output:0"
      input: "ParseSingleExample/ParseExample/ParseExampleV2/sparse_keys:output:0"
      input: "ParseSingleExample/ParseExample/ParseExampleV2/dense_keys:output:0"
      input: "ParseSingleExample/ParseExample/ParseExampleV2/ragged_keys:output:0"
      input: "ParseSingleExample/ParseExample/Const:output:0"
      input: "ParseSingleExample/ParseExample/Const_1:output:0"
      input: "ParseSingleExample/ParseExample/Const_2:output:0"
      attr {
        key: "Tdense"
        value {
          list {
            type: DT_INT64
            type: DT_INT64
            type: DT_INT64
          }
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
                size: 512
              }
            }
            shape {
              dim {
                size: 512
              }
            }
            shape {
              dim {
                size: 512
              }
            }
          }
        }
      }
      attr {
        key: "dense_shapes"
        value {
          list {
            shape {
              dim {
                size: 512
              }
            }
            shape {
              dim {
                size: 512
              }
            }
            shape {
              dim {
                size: 512
              }
            }
          }
        }
      }
      attr {
        key: "num_sparse"
        value {
          i: 0
        }
      }
      attr {
        key: "ragged_split_types"
        value {
          list {
          }
        }
      }
      attr {
        key: "ragged_value_types"
        value {
          list {
          }
        }
      }
      attr {
        key: "sparse_types"
        value {
          list {
          }
        }
      }
      experimental_debug_info {
        original_node_names: "ParseSingleExample/ParseExample/ParseExampleV2"
      }
    }
    node_def {
      name: "Cast"
      op: "Cast"
      input: "ParseSingleExample/ParseExample/ParseExampleV2:dense_values:0"
      attr {
        key: "DstT"
        value {
          type: DT_INT32
        }
      }
      attr {
        key: "SrcT"
        value {
          type: DT_INT64
        }
      }
      attr {
        key: "Truncate"
        value {
          b: false
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
                size: 512
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "Cast"
      }
    }
    node_def {
      name: "Cast_1"
      op: "Cast"
      input: "ParseSingleExample/ParseExample/ParseExampleV2:dense_values:1"
      attr {
        key: "DstT"
        value {
          type: DT_INT32
        }
      }
      attr {
        key: "SrcT"
        value {
          type: DT_INT64
        }
      }
      attr {
        key: "Truncate"
        value {
          b: false
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
                size: 512
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "Cast_1"
      }
    }
    node_def {
      name: "Cast_2"
      op: "Cast"
      input: "ParseSingleExample/ParseExample/ParseExampleV2:dense_values:2"
      attr {
        key: "DstT"
        value {
          type: DT_INT32
        }
      }
      attr {
        key: "SrcT"
        value {
          type: DT_INT64
        }
      }
      attr {
        key: "Truncate"
        value {
          b: false
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
                size: 512
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "Cast_2"
      }
    }
    node_def {
      name: "Identity"
      op: "Identity"
      input: "Cast:y:0"
      attr {
        key: "T"
        value {
          type: DT_INT32
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
                size: 512
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "Identity"
      }
    }
    node_def {
      name: "Identity_1"
      op: "Identity"
      input: "Cast_1:y:0"
      attr {
        key: "T"
        value {
          type: DT_INT32
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
                size: 512
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "Identity_1"
      }
    }
    node_def {
      name: "Identity_2"
      op: "Identity"
      input: "Cast_2:y:0"
      attr {
        key: "T"
        value {
          type: DT_INT32
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
                size: 512
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "Identity_2"
      }
    }
    ret {
      key: "identity"
      value: "Identity:output:0"
    }
    ret {
      key: "identity_1"
      value: "Identity_1:output:0"
    }
    ret {
      key: "identity_2"
      value: "Identity_2:output:0"
    }
    attr {
      key: "_input_shapes"
      value {
        list {
          shape {
          }
        }
      }
    }
    attr {
      key: "_tf_data_function"
      value {
        b: true
      }
    }
    arg_attr {
      value {
        attr {
          key: "_output_shapes"
          value {
            list {
              shape {
              }
            }
          }
        }
        attr {
          key: "_user_specified_name"
          value {
            s: "args_0"
          }
        }
      }
    }
  }
  function {
    signature {
      name: "__inference_tf_data_experimental_parallel_interleave_classabc.ABCMeta_44"
      input_arg {
        name: "args_0"
        type: DT_STRING
      }
      output_arg {
        name: "identity"
        type: DT_VARIANT
      }
      is_stateful: true
      control_output: "TensorSliceDataset"
    }
    node_def {
      name: "flat_filenames/shape"
      op: "Const"
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
                size: 1
              }
            }
          }
        }
      }
      attr {
        key: "dtype"
        value {
          type: DT_INT32
        }
      }
      attr {
        key: "value"
        value {
          tensor {
            dtype: DT_INT32
            tensor_shape {
              dim {
                size: 1
              }
            }
            int_val: -1
          }
        }
      }
      experimental_debug_info {
        original_node_names: "flat_filenames/shape"
      }
    }
    node_def {
      name: "flat_filenames"
      op: "Reshape"
      input: "args_0"
      input: "flat_filenames/shape:output:0"
      attr {
        key: "T"
        value {
          type: DT_STRING
        }
      }
      attr {
        key: "Tshape"
        value {
          type: DT_INT32
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
              dim {
                size: 1
              }
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "flat_filenames"
      }
    }
    node_def {
      name: "TensorSliceDataset"
      op: "TensorSliceDataset"
      input: "flat_filenames:output:0"
      attr {
        key: "Toutput_types"
        value {
          list {
            type: DT_STRING
          }
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
            }
          }
        }
      }
      attr {
        key: "output_shapes"
        value {
          list {
            shape {
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "TensorSliceDataset"
      }
    }
    node_def {
      name: "FlatMapDataset"
      op: "FlatMapDataset"
      input: "TensorSliceDataset:handle:0"
      attr {
        key: "Targuments"
        value {
          list {
          }
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
            }
          }
        }
      }
      attr {
        key: "f"
        value {
          func {
            name: "__inference_Dataset_flat_map_read_one_file_41"
            attr {
              key: "_tf_data_function"
              value {
                b: true
              }
            }
          }
        }
      }
      attr {
        key: "output_shapes"
        value {
          list {
            shape {
            }
          }
        }
      }
      attr {
        key: "output_types"
        value {
          list {
            type: DT_STRING
          }
        }
      }
      experimental_debug_info {
        original_node_names: "FlatMapDataset"
      }
    }
    node_def {
      name: "Identity"
      op: "Identity"
      input: "FlatMapDataset:handle:0"
      input: "^NoOp"
      attr {
        key: "T"
        value {
          type: DT_VARIANT
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
            shape {
            }
          }
        }
      }
      experimental_debug_info {
        original_node_names: "Identity"
      }
    }
    node_def {
      name: "NoOp"
      op: "NoOp"
      input: "^TensorSliceDataset"
      attr {
        key: "_acd_function_control_output"
        value {
          b: true
        }
      }
      attr {
        key: "_output_shapes"
        value {
          list {
          }
        }
      }
      experimental_debug_info {
        original_node_names: "NoOp"
      }
    }
    ret {
      key: "identity"
      value: "Identity:output:0"
    }
    attr {
      key: "_input_shapes"
      value {
        list {
          shape {
          }
        }
      }
    }
    attr {
      key: "_tf_data_function"
      value {
        b: true
      }
    }
    control_ret {
      key: "TensorSliceDataset"
      value: "TensorSliceDataset"
    }
    arg_attr {
      value {
        attr {
          key: "_output_shapes"
          value {
            list {
              shape {
              }
            }
          }
        }
        attr {
          key: "_user_specified_name"
          value {
            s: "args_0"
          }
        }
      }
    }
  }
}
versions {
  producer: 776
  min_consumer: 12
}